From 9b1cb1ced9acc11bc914b1bede7748ef9ca8bef8 Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Fri, 1 May 2026 08:32:46 +0200 Subject: [PATCH] Feature: Add cursor position and type detection for macOS client --- macos/ScreenHandler.h | 6 ++ macos/ScreenHandler.mm | 129 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 6 deletions(-) diff --git a/macos/ScreenHandler.h b/macos/ScreenHandler.h index dea7062..286f3ea 100644 --- a/macos/ScreenHandler.h +++ b/macos/ScreenHandler.h @@ -93,6 +93,12 @@ private: // Get current time in milliseconds static uint64_t getTickMs(); + // Get current cursor position (in physical pixels) + void getCursorPosition(int32_t& x, int32_t& y); + + // Get current cursor type index (matches Windows cursor indices) + uint8_t getCursorTypeIndex(); + private: IOCPClient* m_client; uint64_t m_clientID; diff --git a/macos/ScreenHandler.mm b/macos/ScreenHandler.mm index de8b9a4..9ada1b7 100644 --- a/macos/ScreenHandler.mm +++ b/macos/ScreenHandler.mm @@ -6,6 +6,7 @@ #import "Permissions.h" #import #import +#import #import // Global client ID (calculated in main.mm) @@ -331,13 +332,14 @@ void ScreenHandler::sendDiffFrame() uint8_t algo = m_algorithm.load(); memcpy(data, &algo, sizeof(uint8_t)); - // Write cursor position (simple 0 for now) - int32_t cursorX = 0, cursorY = 0; + // Write cursor position + int32_t cursorX, cursorY; + getCursorPosition(cursorX, cursorY); memcpy(data + 1, &cursorX, sizeof(int32_t)); memcpy(data + 1 + sizeof(int32_t), &cursorY, sizeof(int32_t)); // Write cursor type - uint8_t cursorType = 0; + uint8_t cursorType = getCursorTypeIndex(); memcpy(data + 1 + 2 * sizeof(int32_t), &cursorType, sizeof(uint8_t)); uint32_t headerSize = 1 + 2 * sizeof(int32_t) + 1; @@ -407,13 +409,14 @@ void ScreenHandler::sendH264Frame(bool keyframe) packet[0] = TOKEN_NEXTSCREEN; packet[1] = ALGORITHM_H264; - // Cursor position (0 for now) - int32_t cursorX = 0, cursorY = 0; + // Cursor position + int32_t cursorX, cursorY; + getCursorPosition(cursorX, cursorY); memcpy(&packet[2], &cursorX, sizeof(int32_t)); memcpy(&packet[2 + sizeof(int32_t)], &cursorY, sizeof(int32_t)); // Cursor type - packet[2 + 2 * sizeof(int32_t)] = 0; + packet[2 + 2 * sizeof(int32_t)] = getCursorTypeIndex(); // H264 data memcpy(&packet[headerSize], encodedData, encodedSize); @@ -516,6 +519,120 @@ uint64_t ScreenHandler::getTickMs() return (now * timebase.numer / timebase.denom) / 1000000; } +// Cached logical cursor position (shared between getCursorPosition and getCursorTypeIndex) +static CGPoint s_cachedLogicalPos = {0, 0}; + +void ScreenHandler::getCursorPosition(int32_t& x, int32_t& y) +{ + // Get cursor position in logical (point) coordinates + CGEventRef event = CGEventCreate(nullptr); + s_cachedLogicalPos = CGEventGetLocation(event); + CFRelease(event); + + // Convert to physical pixel coordinates (for Retina displays) + x = (int32_t)(s_cachedLogicalPos.x * m_scaleFactor); + y = (int32_t)(s_cachedLogicalPos.y * m_scaleFactor); + + // Clamp to screen bounds + if (x < 0) x = 0; + if (y < 0) y = 0; + if (x >= m_width) x = m_width - 1; + if (y >= m_height) y = m_height - 1; +} + +uint8_t ScreenHandler::getCursorTypeIndex() +{ + // Windows cursor type indices (from CursorInfo.h): + // 0: IDC_APPSTARTING, 1: IDC_ARROW, 2: IDC_CROSS, 3: IDC_HAND, + // 4: IDC_HELP, 5: IDC_IBEAM, 6: IDC_ICON, 7: IDC_NO, + // 8: IDC_SIZE, 9: IDC_SIZEALL, 10: IDC_SIZENESW, 11: IDC_SIZENS, + // 12: IDC_SIZENWSE, 13: IDC_SIZEWE, 14: IDC_UPARROW, 15: IDC_WAIT + + // NSCursor.currentSystemCursor doesn't work for background daemons. + // Use Accessibility API to infer cursor type from the UI element under cursor. + // Throttle to avoid performance impact (check every 100ms) + + static uint8_t cachedIndex = 1; + static uint64_t lastCheckTime = 0; + static CGPoint lastPos = {-1, -1}; + + // Reuse cursor position from getCursorPosition (called before this) + CGPoint pos = s_cachedLogicalPos; + + // Throttle: only check if cursor moved significantly or 100ms elapsed + uint64_t now = getTickMs(); + bool posChanged = (fabs(pos.x - lastPos.x) > 5 || fabs(pos.y - lastPos.y) > 5); + if (!posChanged && (now - lastCheckTime) < 100) { + return cachedIndex; + } + lastCheckTime = now; + lastPos = pos; + + uint8_t index = 1; // Default to arrow + + // Get the UI element at cursor position using Accessibility API + AXUIElementRef systemWide = AXUIElementCreateSystemWide(); + AXUIElementRef element = nullptr; + + AXError err = AXUIElementCopyElementAtPosition(systemWide, (float)pos.x, (float)pos.y, &element); + CFRelease(systemWide); + + if (err == kAXErrorSuccess && element) { + // Get the role of the element + CFTypeRef roleRef = nullptr; + if (AXUIElementCopyAttributeValue(element, kAXRoleAttribute, &roleRef) == kAXErrorSuccess && roleRef) { + NSString* role = (__bridge NSString*)roleRef; + + // Map UI element roles to cursor types + if ([role isEqualToString:NSAccessibilityTextFieldRole] || + [role isEqualToString:NSAccessibilityTextAreaRole] || + [role isEqualToString:NSAccessibilityStaticTextRole] || + [role isEqualToString:@"AXWebArea"]) { + // Check if text is editable + CFTypeRef editableRef = nullptr; + if (AXUIElementCopyAttributeValue(element, CFSTR("AXEditable"), &editableRef) == kAXErrorSuccess) { + if (editableRef && CFBooleanGetValue((CFBooleanRef)editableRef)) { + index = 5; // IDC_IBEAM for editable text + } + if (editableRef) CFRelease(editableRef); + } else if ([role isEqualToString:NSAccessibilityTextFieldRole] || + [role isEqualToString:NSAccessibilityTextAreaRole]) { + index = 5; // IDC_IBEAM for text input fields + } + } else if ([role isEqualToString:NSAccessibilityLinkRole] || + [role isEqualToString:@"AXLink"]) { + index = 3; // IDC_HAND for links + } else if ([role isEqualToString:NSAccessibilityButtonRole]) { + index = 3; // IDC_HAND for buttons (clickable) + } else if ([role isEqualToString:NSAccessibilitySplitterRole] || + [role isEqualToString:@"AXSplitGroup"]) { + // Check orientation for resize cursor + CFTypeRef orientRef = nullptr; + if (AXUIElementCopyAttributeValue(element, CFSTR("AXOrientation"), &orientRef) == kAXErrorSuccess && orientRef) { + NSString* orient = (__bridge NSString*)orientRef; + if ([orient isEqualToString:@"AXHorizontalOrientation"]) { + index = 11; // IDC_SIZENS (vertical resize) + } else { + index = 13; // IDC_SIZEWE (horizontal resize) + } + CFRelease(orientRef); + } else { + index = 13; // IDC_SIZEWE default for splitters + } + } else if ([role isEqualToString:NSAccessibilityGrowAreaRole]) { + index = 12; // IDC_SIZENWSE for resize corners + } + + CFRelease(roleRef); + } + CFRelease(element); + } + + // Cache the result + cachedIndex = index; + return index; +} + void ScreenHandler::captureLoop() { NSLog(@"ScreenHandler CaptureLoop started (%dx%d)", m_width, m_height);