Feat: Android client Phase 0-3 full implementation
This commit was merged in pull request #3.
This commit is contained in:
360
android/app/src/main/java/com/yama/client/CaptureService.kt
Normal file
360
android/app/src/main/java/com/yama/client/CaptureService.kt
Normal file
@@ -0,0 +1,360 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.app.Notification
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.app.Service
|
||||
import android.content.Intent
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.hardware.display.VirtualDisplay
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaCodecInfo
|
||||
import android.media.MediaFormat
|
||||
import android.media.projection.MediaProjection
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.IBinder
|
||||
import android.os.Looper
|
||||
import android.provider.Settings
|
||||
import android.util.DisplayMetrics
|
||||
import android.util.Log
|
||||
import android.view.WindowManager
|
||||
|
||||
class CaptureService : Service() {
|
||||
|
||||
companion object {
|
||||
const val TAG = "YAMA"
|
||||
const val CHANNEL_ID = "yama_service"
|
||||
const val NOTIF_ID = 1
|
||||
|
||||
const val EXTRA_SERVER_IP = "server_ip"
|
||||
const val EXTRA_SERVER_PORT = "server_port"
|
||||
const val EXTRA_PROJECTION_RESULT = "projection_result"
|
||||
const val EXTRA_PROJECTION_DATA = "projection_data"
|
||||
|
||||
private const val MAX_LONG_SIDE = 1080
|
||||
|
||||
// CS-01 fix: 通过 idrHandler.post 序列化到主线程,避免 TOCTOU 竞争
|
||||
@Volatile var instance: CaptureService? = null
|
||||
|
||||
@JvmStatic
|
||||
fun requestIdr() {
|
||||
val svc = instance ?: return
|
||||
svc.idrHandler.post {
|
||||
runCatching {
|
||||
val p = Bundle()
|
||||
p.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0)
|
||||
svc.mediaCodec?.setParameters(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 静止屏幕黑屏修复:SurfaceFlinger 在无脏区时不向 VirtualDisplay 推帧,
|
||||
* 导致编码器无输入,REQUEST_SYNC_FRAME 悬空排队,浏览器等 10-60 秒才见画面。
|
||||
* 解决方法:将 VirtualDisplay 宽度临时改变 2px,触发一次强制合成,
|
||||
* 编码器随即得到输入帧并消费已排队的 IDR 请求。150ms 后恢复原始尺寸。
|
||||
*/
|
||||
@JvmStatic
|
||||
fun forceFirstFrame() {
|
||||
val svc = instance ?: return
|
||||
svc.idrHandler.post {
|
||||
runCatching {
|
||||
val vd = svc.virtualDisplay ?: return@runCatching
|
||||
val w = ControlService.encW
|
||||
val h = ControlService.encH
|
||||
val dpi = svc.resources.displayMetrics.densityDpi
|
||||
Log.d(TAG, "forceFirstFrame: resize ${w}x${h} → ${w+2}x${h} to trigger composition")
|
||||
vd.resize(w + 2, h, dpi)
|
||||
svc.idrHandler.postDelayed({
|
||||
runCatching { vd.resize(w, h, dpi) }
|
||||
Log.d(TAG, "forceFirstFrame: restored ${w}x${h}")
|
||||
}, 150)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var mediaProjection: MediaProjection? = null
|
||||
private var virtualDisplay: VirtualDisplay? = null
|
||||
private var mediaCodec: MediaCodec? = null
|
||||
|
||||
private val idrHandler = Handler(Looper.getMainLooper())
|
||||
// CS-04 fix: runCatching 包住 setParameters,防止 codec 已停止时抛异常
|
||||
private val idrRunnable = object : Runnable {
|
||||
override fun run() {
|
||||
runCatching {
|
||||
val p = Bundle()
|
||||
p.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0)
|
||||
mediaCodec?.setParameters(p)
|
||||
}
|
||||
idrHandler.postDelayed(this, 1000)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
instance = this
|
||||
createNotificationChannel()
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIF_ID, buildNotification(),
|
||||
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
|
||||
} else {
|
||||
startForeground(NOTIF_ID, buildNotification())
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
// CS-03 fix: START_STICKY 重启时 intent 为 null,必须在最前面检查
|
||||
if (intent == null) return START_NOT_STICKY
|
||||
val ip = intent.getStringExtra(EXTRA_SERVER_IP) ?: return START_NOT_STICKY
|
||||
val port = intent.getIntExtra(EXTRA_SERVER_PORT, 443)
|
||||
|
||||
val androidId = Settings.Secure.getString(contentResolver, Settings.Secure.ANDROID_ID) ?: ""
|
||||
val model = "${Build.MANUFACTURER} ${Build.MODEL}"
|
||||
val osVer = "Android ${Build.VERSION.RELEASE} (API ${Build.VERSION.SDK_INT})"
|
||||
val (sw, sh) = getPhysicalResolution()
|
||||
val res = "1:${sw}*${sh}"
|
||||
val user = Build.USER
|
||||
|
||||
Log.i(TAG, "CaptureService: server=$ip:$port device=$model res=$res")
|
||||
|
||||
// YB-01 fix: so 库加载失败是 Error,不被 catch(Exception) 捕获,需单独处理
|
||||
try {
|
||||
YamaBridge.nativeInit(ip, port, androidId, model, osVer, res, user, packageCodePath ?: "")
|
||||
} catch (e: UnsatisfiedLinkError) {
|
||||
Log.e(TAG, "Native library load failed: $e")
|
||||
stopSelf()
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
|
||||
val projResult = intent.getIntExtra(EXTRA_PROJECTION_RESULT, -1)
|
||||
val projData = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU)
|
||||
intent.getParcelableExtra(EXTRA_PROJECTION_DATA, Intent::class.java)
|
||||
else
|
||||
@Suppress("DEPRECATION") intent.getParcelableExtra(EXTRA_PROJECTION_DATA)
|
||||
|
||||
if (projResult == android.app.Activity.RESULT_OK && projData != null) {
|
||||
val pm = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val mp = pm.getMediaProjection(projResult, projData)
|
||||
mediaProjection = mp
|
||||
// CS-05 fix: 分辨率为 0 时不启动采集,避免 MediaCodec 配置崩溃
|
||||
if (sw > 0 && sh > 0) {
|
||||
startCapture(mp, sw, sh)
|
||||
} else {
|
||||
Log.e(TAG, "Invalid screen resolution ${sw}x${sh}, skipping capture")
|
||||
}
|
||||
} else {
|
||||
Log.w(TAG, "No MediaProjection token, skipping screen capture")
|
||||
}
|
||||
|
||||
return START_STICKY
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
instance = null
|
||||
stopCapture()
|
||||
YamaBridge.nativeStop()
|
||||
Log.i(TAG, "CaptureService destroyed")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- MediaProjection + MediaCodec
|
||||
|
||||
private fun startCapture(mp: MediaProjection, physW: Int, physH: Int) {
|
||||
val (encW, encH) = scaleDown(physW, physH, MAX_LONG_SIDE)
|
||||
val dpi = resources.displayMetrics.densityDpi
|
||||
|
||||
Log.i(TAG, "Capture: phys=${physW}x${physH} enc=${encW}x${encH} dpi=$dpi")
|
||||
|
||||
val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
|
||||
val surface = configureEncoder(codec, encW, encH)
|
||||
|
||||
var configData: ByteArray? = null
|
||||
|
||||
codec.setCallback(object : MediaCodec.Callback() {
|
||||
override fun onInputBufferAvailable(mc: MediaCodec, index: Int) { /* Surface 模式不用 */ }
|
||||
|
||||
override fun onOutputBufferAvailable(mc: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
|
||||
val buf = mc.getOutputBuffer(index)
|
||||
if (buf == null || info.size == 0) {
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
return
|
||||
}
|
||||
buf.position(info.offset)
|
||||
buf.limit(info.offset + info.size)
|
||||
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
|
||||
configData = ByteArray(info.size).also { buf.get(it) }
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
return
|
||||
}
|
||||
|
||||
val isKey = info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
|
||||
val frame = ByteArray(info.size).also { buf.get(it) }
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
|
||||
// 部分安卓硬件编码器(如某些高通实现)对 REQUEST_SYNC_FRAME 强制产生的
|
||||
// IDR 帧不设置 BUFFER_FLAG_KEY_FRAME,导致 SendLoop 将所有帧视为 P 帧丢弃。
|
||||
// 补充扫描 NALU 字节来识别真正的关键帧,确保首帧能顺利发出。
|
||||
val actualIsKey = isKey || isNaluKeyframe(frame)
|
||||
val payload = if (actualIsKey && configData != null) configData!! + frame else frame
|
||||
YamaBridge.nativeOnH264Frame(payload, 0, payload.size, actualIsKey)
|
||||
}
|
||||
|
||||
override fun onError(mc: MediaCodec, e: MediaCodec.CodecException) {
|
||||
Log.e(TAG, "MediaCodec error: $e")
|
||||
}
|
||||
|
||||
override fun onOutputFormatChanged(mc: MediaCodec, format: MediaFormat) {
|
||||
Log.i(TAG, "Output format: $format")
|
||||
}
|
||||
})
|
||||
|
||||
codec.start()
|
||||
mediaCodec = codec
|
||||
|
||||
idrHandler.postDelayed(idrRunnable, 500)
|
||||
|
||||
// Android 14+ requires callback registered before createVirtualDisplay()
|
||||
mp.registerCallback(object : MediaProjection.Callback() {
|
||||
override fun onStop() {
|
||||
Log.i(TAG, "MediaProjection stopped")
|
||||
stopCapture()
|
||||
stopSelf()
|
||||
}
|
||||
}, null)
|
||||
|
||||
virtualDisplay = mp.createVirtualDisplay(
|
||||
"YAMA",
|
||||
encW, encH, dpi,
|
||||
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
||||
surface, null, null
|
||||
)
|
||||
|
||||
YamaBridge.nativeSetScreenSize(encW, encH)
|
||||
|
||||
ControlService.physW = physW
|
||||
ControlService.physH = physH
|
||||
ControlService.encW = encW
|
||||
ControlService.encH = encH
|
||||
|
||||
Log.i(TAG, "MediaCodec + VirtualDisplay started ${encW}x${encH}")
|
||||
}
|
||||
|
||||
private fun buildFmt(w: Int, h: Int, lowLatency: Boolean, baseline: Boolean) =
|
||||
MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, w, h).apply {
|
||||
setInteger(MediaFormat.KEY_COLOR_FORMAT,
|
||||
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
|
||||
setInteger(MediaFormat.KEY_BIT_RATE, 4_000_000)
|
||||
setInteger(MediaFormat.KEY_FRAME_RATE, 30)
|
||||
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 2)
|
||||
setInteger(MediaFormat.KEY_PRIORITY, 0)
|
||||
setInteger(MediaFormat.KEY_OPERATING_RATE, 30)
|
||||
if (lowLatency) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N)
|
||||
setInteger(MediaFormat.KEY_LATENCY, 0)
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q)
|
||||
setInteger(MediaFormat.KEY_LOW_LATENCY, 1)
|
||||
}
|
||||
if (baseline)
|
||||
setInteger(MediaFormat.KEY_PROFILE,
|
||||
MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline)
|
||||
}
|
||||
|
||||
// 逐级降级:低延迟+Baseline → 低延迟 → 默认,确保真机兼容
|
||||
private fun configureEncoder(codec: MediaCodec, w: Int, h: Int): android.view.Surface {
|
||||
val configs = listOf(
|
||||
true to true,
|
||||
true to false,
|
||||
false to false,
|
||||
)
|
||||
for ((ll, bl) in configs) {
|
||||
try {
|
||||
val fmt = buildFmt(w, h, ll, bl)
|
||||
codec.configure(fmt, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
||||
Log.i(TAG, "Encoder configured: lowLatency=$ll baseline=$bl")
|
||||
return codec.createInputSurface()
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Encoder config failed (lowLatency=$ll baseline=$bl): $e, retrying…")
|
||||
try { codec.reset() } catch (_: Exception) {}
|
||||
}
|
||||
}
|
||||
throw RuntimeException("MediaCodec: all encoder configs failed")
|
||||
}
|
||||
|
||||
private fun stopCapture() {
|
||||
idrHandler.removeCallbacks(idrRunnable)
|
||||
runCatching { virtualDisplay?.release() }
|
||||
// CS-07 fix: stop 和 release 分开,stop 失败不会跳过 release 导致资源泄漏
|
||||
runCatching { mediaCodec?.stop() }
|
||||
runCatching { mediaCodec?.release() }
|
||||
runCatching { mediaProjection?.stop() }
|
||||
virtualDisplay = null
|
||||
mediaCodec = null
|
||||
mediaProjection = null
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 辅助函数
|
||||
|
||||
private fun getPhysicalResolution(): Pair<Int, Int> {
|
||||
val wm = getSystemService(WINDOW_SERVICE) as WindowManager
|
||||
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
val b = wm.currentWindowMetrics.bounds
|
||||
Pair(b.width(), b.height())
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
val dm = DisplayMetrics()
|
||||
@Suppress("DEPRECATION")
|
||||
wm.defaultDisplay.getRealMetrics(dm)
|
||||
Pair(dm.widthPixels, dm.heightPixels)
|
||||
}
|
||||
}
|
||||
|
||||
/** 扫描 Annex-B NALU 流,遇到 NAL type 5(IDR)、7(SPS)、8(PPS) 即判定为关键帧 */
|
||||
private fun isNaluKeyframe(data: ByteArray): Boolean {
|
||||
var i = 0
|
||||
while (i + 4 < data.size) {
|
||||
if (data[i] == 0.toByte() && data[i+1] == 0.toByte() &&
|
||||
data[i+2] == 0.toByte() && data[i+3] == 1.toByte()) {
|
||||
val nalType = data[i + 4].toInt() and 0x1F
|
||||
if (nalType == 5 || nalType == 7 || nalType == 8) return true
|
||||
i += 5
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 按长边上限等比缩小,宽高各自 2 对齐(H.264 要求) */
|
||||
private fun scaleDown(w: Int, h: Int, maxLong: Int): Pair<Int, Int> {
|
||||
val longSide = maxOf(w, h)
|
||||
if (longSide <= maxLong) return Pair(w and -2, h and -2)
|
||||
val scale = maxLong.toDouble() / longSide
|
||||
return Pair((w * scale).toInt() and -2, (h * scale).toInt() and -2)
|
||||
}
|
||||
|
||||
private fun createNotificationChannel() {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
val ch = NotificationChannel(CHANNEL_ID, "YAMA Service",
|
||||
NotificationManager.IMPORTANCE_LOW)
|
||||
getSystemService(NotificationManager::class.java)?.createNotificationChannel(ch)
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildNotification(): Notification {
|
||||
val b = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)
|
||||
Notification.Builder(this, CHANNEL_ID)
|
||||
else
|
||||
@Suppress("DEPRECATION") Notification.Builder(this)
|
||||
return b.setContentTitle("YAMA")
|
||||
.setContentText("Screen monitoring active")
|
||||
.setSmallIcon(android.R.drawable.ic_menu_camera)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
290
android/app/src/main/java/com/yama/client/ControlService.kt
Normal file
290
android/app/src/main/java/com/yama/client/ControlService.kt
Normal file
@@ -0,0 +1,290 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.accessibilityservice.AccessibilityService
|
||||
import android.accessibilityservice.GestureDescription
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.BroadcastReceiver
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.IntentFilter
|
||||
import android.graphics.Path
|
||||
import android.graphics.PointF
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import android.view.accessibility.AccessibilityEvent
|
||||
|
||||
/**
|
||||
* AccessibilityService that injects touch gestures and global key actions
|
||||
* dispatched by the server (COMMAND_SCREEN_CONTROL / MSG64).
|
||||
*
|
||||
* Requires API 24+ for dispatchGesture(); guarded at runtime so the APK
|
||||
* installs on API 21+ but gesture injection only runs on 24+.
|
||||
*
|
||||
* Activation: Settings → Accessibility → YAMA → enable.
|
||||
*/
|
||||
@SuppressLint("NewApi")
|
||||
class ControlService : AccessibilityService() {
|
||||
|
||||
companion object {
|
||||
const val TAG = "YAMA_CTRL"
|
||||
|
||||
// ── Service instance ─────────────────────────────────────────────
|
||||
@Volatile var instance: ControlService? = null
|
||||
|
||||
// ── Active foreground window ─────────────────────────────────────
|
||||
@JvmStatic
|
||||
@Volatile var activeWindow: String = "Android"
|
||||
|
||||
// ── Screen geometry (set by CaptureService) ──────────────────────
|
||||
// Server sends coords in encoding space [0, encW) × [0, encH).
|
||||
// We map them to physical screen space before dispatching.
|
||||
@Volatile var physW = 1080
|
||||
@Volatile var physH = 1920
|
||||
@Volatile var encW = 1080
|
||||
@Volatile var encH = 1920
|
||||
|
||||
// ── Windows message constants ────────────────────────────────────
|
||||
const val WM_MOUSEMOVE = 0x0200
|
||||
const val WM_LBUTTONDOWN = 0x0201
|
||||
const val WM_LBUTTONUP = 0x0202
|
||||
const val WM_LBUTTONDBLCLK = 0x0203
|
||||
const val WM_RBUTTONDOWN = 0x0204
|
||||
const val WM_MBUTTONDOWN = 0x0207
|
||||
const val WM_MOUSEWHEEL = 0x020A
|
||||
const val WM_KEYDOWN = 0x0100
|
||||
const val WM_SYSKEYDOWN = 0x0104
|
||||
|
||||
/**
|
||||
* Called from C++ DataProcess() via JNI on the IO thread.
|
||||
* Parameters:
|
||||
* message – Windows WM_* constant (low 16 bits of MSG64.message)
|
||||
* wParam – MSG64.wParam (key/button flags; high word of MOUSEWHEEL = delta)
|
||||
* ptX/ptY – MSG64.pt in encoding-space pixels
|
||||
*/
|
||||
@JvmStatic
|
||||
fun onControlEvent(message: Int, wParam: Long, ptX: Int, ptY: Int) {
|
||||
val svc = instance ?: run {
|
||||
Log.w(TAG, "ControlService inactive — event 0x${message.toString(16)} dropped")
|
||||
return
|
||||
}
|
||||
// All gesture state lives on the main thread; post there.
|
||||
svc.handler.post { svc.handleEvent(message, wParam, ptX, ptY) }
|
||||
}
|
||||
}
|
||||
|
||||
// ── Screen state receiver ─────────────────────────────────────────────
|
||||
private val screenReceiver = object : BroadcastReceiver() {
|
||||
override fun onReceive(context: Context?, intent: Intent?) {
|
||||
when (intent?.action) {
|
||||
Intent.ACTION_SCREEN_OFF -> activeWindow = "Locked"
|
||||
Intent.ACTION_USER_PRESENT -> activeWindow = "Android"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main-thread handler ───────────────────────────────────────────────
|
||||
private val handler = Handler(Looper.getMainLooper())
|
||||
|
||||
// ── Left-button drag accumulator ─────────────────────────────────────
|
||||
private var lbuttonDown = false
|
||||
private var gestureStart = 0L
|
||||
// Pairs of (millisecond offset from gesture start, screen point)
|
||||
private val strokePoints = mutableListOf<Pair<Long, PointF>>()
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// AccessibilityService lifecycle
|
||||
|
||||
override fun onServiceConnected() {
|
||||
super.onServiceConnected()
|
||||
instance = this
|
||||
val filter = IntentFilter().apply {
|
||||
addAction(Intent.ACTION_SCREEN_OFF)
|
||||
addAction(Intent.ACTION_USER_PRESENT)
|
||||
}
|
||||
registerReceiver(screenReceiver, filter)
|
||||
Log.i(TAG, "ControlService connected phys=${physW}x${physH} enc=${encW}x${encH}")
|
||||
}
|
||||
|
||||
override fun onAccessibilityEvent(event: AccessibilityEvent?) {
|
||||
if (event?.eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) {
|
||||
val pkg = event.packageName?.toString()
|
||||
if (!pkg.isNullOrEmpty()) activeWindow = pkg
|
||||
}
|
||||
}
|
||||
override fun onInterrupt() {}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
runCatching { unregisterReceiver(screenReceiver) }
|
||||
instance = null
|
||||
Log.i(TAG, "ControlService destroyed")
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// Event routing (runs on main thread)
|
||||
|
||||
private fun handleEvent(message: Int, wParam: Long, ptX: Int, ptY: Int) {
|
||||
// Scale encoding-space coords to physical screen
|
||||
val x = if (encW > 0) ptX.toFloat() * physW / encW else ptX.toFloat()
|
||||
val y = if (encH > 0) ptY.toFloat() * physH / encH else ptY.toFloat()
|
||||
if (message != WM_MOUSEMOVE)
|
||||
Log.d(TAG, "event 0x${message.toString(16)} enc=($ptX,$ptY) phys=(${x.toInt()},${y.toInt()})")
|
||||
|
||||
when (message) {
|
||||
|
||||
// ── Left button: accumulate drag path, dispatch on up ─────────
|
||||
WM_LBUTTONDOWN -> {
|
||||
lbuttonDown = true
|
||||
gestureStart = SystemClock.uptimeMillis()
|
||||
strokePoints.clear()
|
||||
strokePoints.add(0L to PointF(x, y))
|
||||
}
|
||||
WM_MOUSEMOVE -> {
|
||||
if (lbuttonDown) {
|
||||
strokePoints.add((SystemClock.uptimeMillis() - gestureStart) to PointF(x, y))
|
||||
}
|
||||
}
|
||||
WM_LBUTTONUP -> {
|
||||
if (!lbuttonDown) return
|
||||
lbuttonDown = false
|
||||
val elapsed = SystemClock.uptimeMillis() - gestureStart
|
||||
strokePoints.add(elapsed to PointF(x, y))
|
||||
val down = strokePoints[0].second
|
||||
val dx = x - down.x
|
||||
val dy = y - down.y
|
||||
|
||||
// ── System edge-gesture detection ────────────────────────────
|
||||
// dispatchGesture() is unreliable for system gestures; use performGlobalAction.
|
||||
val hEdge = physH * 0.08f // 8% of screen height = top/bottom trigger zone
|
||||
val wEdge = physW * 0.08f // 8% of screen width = left/right trigger zone
|
||||
val minH = physH * 0.15f // minimum vertical travel
|
||||
val minW = physW * 0.20f // minimum horizontal travel
|
||||
|
||||
val fromLeft = down.x < wEdge && dx > minW && Math.abs(dx) > Math.abs(dy) * 1.5f && elapsed < 600L
|
||||
val fromRight = down.x > physW - wEdge && dx < -minW && Math.abs(dx) > Math.abs(dy) * 1.5f && elapsed < 600L
|
||||
val fromTop = down.y < hEdge && dy > minH && Math.abs(dy) > Math.abs(dx)
|
||||
val fromBottom = down.y > physH - hEdge && dy < -minH && Math.abs(dy) > Math.abs(dx)
|
||||
|
||||
when {
|
||||
fromLeft || fromRight -> {
|
||||
Log.d(TAG, "Edge-swipe back (fromLeft=$fromLeft)")
|
||||
performGlobalAction(GLOBAL_ACTION_BACK)
|
||||
}
|
||||
fromTop -> {
|
||||
// Right half of screen → quick settings; left half → notifications
|
||||
if (down.x > physW * 0.5f) {
|
||||
Log.d(TAG, "Top-swipe quick settings")
|
||||
performGlobalAction(GLOBAL_ACTION_QUICK_SETTINGS)
|
||||
} else {
|
||||
Log.d(TAG, "Top-swipe notifications")
|
||||
performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS)
|
||||
}
|
||||
}
|
||||
fromBottom -> {
|
||||
// Slow swipe (≥400 ms) or very long travel → Recents; quick swipe → Home
|
||||
if (elapsed >= 400L || Math.abs(dy) > physH * 0.40f) {
|
||||
Log.d(TAG, "Bottom-swipe recents (elapsed=${elapsed}ms)")
|
||||
performGlobalAction(GLOBAL_ACTION_RECENTS)
|
||||
} else {
|
||||
Log.d(TAG, "Bottom-swipe home (elapsed=${elapsed}ms)")
|
||||
performGlobalAction(GLOBAL_ACTION_HOME)
|
||||
}
|
||||
}
|
||||
dx * dx + dy * dy < 100f -> // 移动 < 10px → 单击
|
||||
dispatchTap(down.x, down.y)
|
||||
else ->
|
||||
dispatchTouchGesture(strokePoints)
|
||||
}
|
||||
// 手势执行后立即请求关键帧,让 decoder 尽快看到屏幕变化
|
||||
CaptureService.requestIdr()
|
||||
strokePoints.clear()
|
||||
}
|
||||
|
||||
// ── Double-click → two rapid taps ────────────────────────────
|
||||
WM_LBUTTONDBLCLK -> {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||||
val p = singlePointPath(x, y)
|
||||
val gesture = GestureDescription.Builder()
|
||||
.addStroke(GestureDescription.StrokeDescription(p, 0L, 80L))
|
||||
.addStroke(GestureDescription.StrokeDescription(p, 200L, 80L))
|
||||
.build()
|
||||
dispatchGesture(gesture, null, null)
|
||||
}
|
||||
|
||||
// ── Right-click → long press (600 ms) ────────────────────────
|
||||
WM_RBUTTONDOWN -> {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||||
val stroke = GestureDescription.StrokeDescription(singlePointPath(x, y), 0L, 600L)
|
||||
dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null)
|
||||
}
|
||||
|
||||
// ── Scroll wheel → vertical swipe ─────────────────────────────
|
||||
WM_MOUSEWHEEL -> {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||||
// High 16 bits of wParam hold the signed delta (+120 = up, -120 = down)
|
||||
val delta = (wParam.toInt() ushr 16).toShort().toInt()
|
||||
// Wheel-up (+delta) = user wants content above = finger swipes DOWN (+y)
|
||||
val dy = if (delta > 0) 400f else -400f
|
||||
val path = Path().apply { moveTo(x, y); lineTo(x, y + dy) }
|
||||
val stroke = GestureDescription.StrokeDescription(path, 0L, 300L)
|
||||
dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null)
|
||||
CaptureService.requestIdr()
|
||||
}
|
||||
|
||||
// ── Keyboard → global actions only ───────────────────────────
|
||||
WM_KEYDOWN, WM_SYSKEYDOWN -> handleKeyDown(wParam.toInt() and 0xFFFF)
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
|
||||
private fun dispatchTouchGesture(points: List<Pair<Long, PointF>>) {
|
||||
if (points.isEmpty()) return
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||||
|
||||
val path = Path()
|
||||
path.moveTo(points[0].second.x, points[0].second.y)
|
||||
for (i in 1 until points.size) {
|
||||
path.lineTo(points[i].second.x, points[i].second.y)
|
||||
}
|
||||
// Minimum 50 ms so Android registers the gesture; use actual elapsed for drags.
|
||||
val duration = maxOf(points.last().first, 50L)
|
||||
val stroke = GestureDescription.StrokeDescription(path, 0L, duration)
|
||||
val ok = dispatchGesture(
|
||||
GestureDescription.Builder().addStroke(stroke).build(),
|
||||
object : GestureResultCallback() {
|
||||
override fun onCompleted(g: GestureDescription?) { Log.d(TAG, "Gesture ok") }
|
||||
override fun onCancelled(g: GestureDescription?) { Log.w(TAG, "Gesture cancelled") }
|
||||
},
|
||||
null
|
||||
)
|
||||
if (!ok) Log.e(TAG, "dispatchGesture returned false (service not ready?)")
|
||||
}
|
||||
|
||||
private fun handleKeyDown(vk: Int) {
|
||||
when (vk) {
|
||||
0x08, 0x1B -> performGlobalAction(GLOBAL_ACTION_BACK) // Backspace / Esc
|
||||
0x24 -> performGlobalAction(GLOBAL_ACTION_HOME) // VK_HOME
|
||||
0x5D -> performGlobalAction(GLOBAL_ACTION_RECENTS) // VK_APPS
|
||||
0x2C -> performGlobalAction(GLOBAL_ACTION_TAKE_SCREENSHOT) // PrtSc
|
||||
}
|
||||
}
|
||||
|
||||
private fun dispatchTap(x: Float, y: Float) {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||||
val path = Path().apply { moveTo(x, y); lineTo(x + 1f, y) }
|
||||
val stroke = GestureDescription.StrokeDescription(path, 0L, 100L)
|
||||
val ok = dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(),
|
||||
object : GestureResultCallback() {
|
||||
override fun onCompleted(g: GestureDescription?) { Log.d(TAG, "Tap ok") }
|
||||
override fun onCancelled(g: GestureDescription?) { Log.w(TAG, "Tap cancelled") }
|
||||
}, null)
|
||||
if (!ok) Log.e(TAG, "dispatchTap returned false")
|
||||
}
|
||||
|
||||
private fun singlePointPath(x: Float, y: Float) = Path().apply { moveTo(x, y); lineTo(x + 1f, y) }
|
||||
}
|
||||
65
android/app/src/main/java/com/yama/client/MainActivity.kt
Normal file
65
android/app/src/main/java/com/yama/client/MainActivity.kt
Normal file
@@ -0,0 +1,65 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.app.Activity
|
||||
import android.content.Intent
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import android.os.Bundle
|
||||
import android.util.Log
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
|
||||
class MainActivity : AppCompatActivity() {
|
||||
|
||||
private val serverIp = "91.99.165.207"
|
||||
private val serverPort = 443
|
||||
|
||||
private lateinit var projectionManager: MediaProjectionManager
|
||||
|
||||
companion object {
|
||||
const val REQUEST_MEDIA_PROJECTION = 1001
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
// MA-01 fix: 系统重建 Activity 时 savedInstanceState != null,
|
||||
// 此时 CaptureService 可能已在运行,不重复弹权限对话框
|
||||
if (savedInstanceState != null) { finish(); return }
|
||||
projectionManager = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
startActivityForResult(projectionManager.createScreenCaptureIntent(), REQUEST_MEDIA_PROJECTION)
|
||||
}
|
||||
|
||||
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
|
||||
super.onActivityResult(requestCode, resultCode, data)
|
||||
if (requestCode == REQUEST_MEDIA_PROJECTION) {
|
||||
if (resultCode == Activity.RESULT_OK && data != null) {
|
||||
Log.i(CaptureService.TAG, "MediaProjection permission granted")
|
||||
val intent = Intent(this, CaptureService::class.java).apply {
|
||||
putExtra(CaptureService.EXTRA_SERVER_IP, serverIp)
|
||||
putExtra(CaptureService.EXTRA_SERVER_PORT, serverPort)
|
||||
putExtra(CaptureService.EXTRA_PROJECTION_RESULT, resultCode)
|
||||
putExtra(CaptureService.EXTRA_PROJECTION_DATA, data)
|
||||
}
|
||||
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.O) {
|
||||
startForegroundService(intent)
|
||||
} else {
|
||||
startService(intent)
|
||||
}
|
||||
} else {
|
||||
Log.w(CaptureService.TAG, "MediaProjection permission denied, starting without capture")
|
||||
val intent = Intent(this, CaptureService::class.java).apply {
|
||||
putExtra(CaptureService.EXTRA_SERVER_IP, serverIp)
|
||||
putExtra(CaptureService.EXTRA_SERVER_PORT, serverPort)
|
||||
}
|
||||
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.O) {
|
||||
startForegroundService(intent)
|
||||
} else {
|
||||
startService(intent)
|
||||
}
|
||||
}
|
||||
finish()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
}
|
||||
}
|
||||
26
android/app/src/main/java/com/yama/client/YamaBridge.kt
Normal file
26
android/app/src/main/java/com/yama/client/YamaBridge.kt
Normal file
@@ -0,0 +1,26 @@
|
||||
package com.yama.client
|
||||
|
||||
object YamaBridge {
|
||||
init {
|
||||
System.loadLibrary("yama")
|
||||
}
|
||||
|
||||
external fun nativeInit(
|
||||
serverIp: String,
|
||||
serverPort: Int,
|
||||
androidId: String,
|
||||
deviceModel: String,
|
||||
androidVersion: String,
|
||||
screenRes: String,
|
||||
username: String,
|
||||
apkPath: String
|
||||
): Int
|
||||
|
||||
external fun nativeStop()
|
||||
|
||||
/** MediaCodec 配置完成后调用,告知 C++ 实际捕获分辨率 */
|
||||
external fun nativeSetScreenSize(width: Int, height: Int)
|
||||
|
||||
/** MediaCodec 每输出一帧 H.264 NALU 时调用 */
|
||||
external fun nativeOnH264Frame(data: ByteArray, offset: Int, size: Int, isKeyframe: Boolean)
|
||||
}
|
||||
Reference in New Issue
Block a user