yuanyuanxiang/SimpleRemoter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Feature: Add cursor position and type detection for macOS client

Browse Source
This commit is contained in:
yuanyuanxiang
2026-05-01 08:32:46 +02:00
parent f2a184e760
commit 9b1cb1ced9
2 changed files with 129 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
macos/ScreenHandler.mm
View File

@@ -6,6 +6,7 @@
#import "Permissions.h"
#import <Cocoa/Cocoa.h>
#import <CoreGraphics/CoreGraphics.h>
#import <ApplicationServices/ApplicationServices.h>
#import <mach/mach_time.h>
// Global client ID (calculated in main.mm)
@@ -331,13 +332,14 @@ void ScreenHandler::sendDiffFrame()
uint8_t algo = m_algorithm.load();
memcpy(data, &algo, sizeof(uint8_t));
// Write cursor position (simple 0 for now)
int32_t cursorX = 0, cursorY = 0;
// Write cursor position
int32_t cursorX, cursorY;
getCursorPosition(cursorX, cursorY);
memcpy(data + 1, &cursorX, sizeof(int32_t));
memcpy(data + 1 + sizeof(int32_t), &cursorY, sizeof(int32_t));
// Write cursor type
uint8_t cursorType = 0;
uint8_t cursorType = getCursorTypeIndex();
memcpy(data + 1 + 2 * sizeof(int32_t), &cursorType, sizeof(uint8_t));
uint32_t headerSize = 1 + 2 * sizeof(int32_t) + 1;
@@ -407,13 +409,14 @@ void ScreenHandler::sendH264Frame(bool keyframe)
packet[0] = TOKEN_NEXTSCREEN;
packet[1] = ALGORITHM_H264;
// Cursor position (0 for now)
int32_t cursorX = 0, cursorY = 0;
// Cursor position
int32_t cursorX, cursorY;
getCursorPosition(cursorX, cursorY);
memcpy(&packet[2], &cursorX, sizeof(int32_t));
memcpy(&packet[2 + sizeof(int32_t)], &cursorY, sizeof(int32_t));
// Cursor type
packet[2 + 2 * sizeof(int32_t)] = 0;
packet[2 + 2 * sizeof(int32_t)] = getCursorTypeIndex();
// H264 data
memcpy(&packet[headerSize], encodedData, encodedSize);
@@ -516,6 +519,120 @@ uint64_t ScreenHandler::getTickMs()
return (now * timebase.numer / timebase.denom) / 1000000;
}
// Cached logical cursor position (shared between getCursorPosition and getCursorTypeIndex)
static CGPoint s_cachedLogicalPos = {0, 0};
void ScreenHandler::getCursorPosition(int32_t& x, int32_t& y)
{
// Get cursor position in logical (point) coordinates
CGEventRef event = CGEventCreate(nullptr);
s_cachedLogicalPos = CGEventGetLocation(event);
CFRelease(event);
// Convert to physical pixel coordinates (for Retina displays)
x = (int32_t)(s_cachedLogicalPos.x * m_scaleFactor);
y = (int32_t)(s_cachedLogicalPos.y * m_scaleFactor);
// Clamp to screen bounds
if (x < 0) x = 0;
if (y < 0) y = 0;
if (x >= m_width) x = m_width - 1;
if (y >= m_height) y = m_height - 1;
}
uint8_t ScreenHandler::getCursorTypeIndex()
{
// Windows cursor type indices (from CursorInfo.h):
// 0: IDC_APPSTARTING, 1: IDC_ARROW, 2: IDC_CROSS, 3: IDC_HAND,
// 4: IDC_HELP, 5: IDC_IBEAM, 6: IDC_ICON, 7: IDC_NO,
// 8: IDC_SIZE, 9: IDC_SIZEALL, 10: IDC_SIZENESW, 11: IDC_SIZENS,
// 12: IDC_SIZENWSE, 13: IDC_SIZEWE, 14: IDC_UPARROW, 15: IDC_WAIT
// NSCursor.currentSystemCursor doesn't work for background daemons.
// Use Accessibility API to infer cursor type from the UI element under cursor.
// Throttle to avoid performance impact (check every 100ms)
static uint8_t cachedIndex = 1;
static uint64_t lastCheckTime = 0;
static CGPoint lastPos = {-1, -1};
// Reuse cursor position from getCursorPosition (called before this)
CGPoint pos = s_cachedLogicalPos;
// Throttle: only check if cursor moved significantly or 100ms elapsed
uint64_t now = getTickMs();
bool posChanged = (fabs(pos.x - lastPos.x) > 5 || fabs(pos.y - lastPos.y) > 5);
if (!posChanged && (now - lastCheckTime) < 100) {
return cachedIndex;
}
lastCheckTime = now;
lastPos = pos;
uint8_t index = 1; // Default to arrow
// Get the UI element at cursor position using Accessibility API
AXUIElementRef systemWide = AXUIElementCreateSystemWide();
AXUIElementRef element = nullptr;
AXError err = AXUIElementCopyElementAtPosition(systemWide, (float)pos.x, (float)pos.y, &element);
CFRelease(systemWide);
if (err == kAXErrorSuccess && element) {
// Get the role of the element
CFTypeRef roleRef = nullptr;
if (AXUIElementCopyAttributeValue(element, kAXRoleAttribute, &roleRef) == kAXErrorSuccess && roleRef) {
NSString* role = (__bridge NSString*)roleRef;
// Map UI element roles to cursor types
if ([role isEqualToString:NSAccessibilityTextFieldRole] ||
[role isEqualToString:NSAccessibilityTextAreaRole] ||
[role isEqualToString:NSAccessibilityStaticTextRole] ||
[role isEqualToString:@"AXWebArea"]) {
// Check if text is editable
CFTypeRef editableRef = nullptr;
if (AXUIElementCopyAttributeValue(element, CFSTR("AXEditable"), &editableRef) == kAXErrorSuccess) {
if (editableRef && CFBooleanGetValue((CFBooleanRef)editableRef)) {
index = 5; // IDC_IBEAM for editable text
}
if (editableRef) CFRelease(editableRef);
} else if ([role isEqualToString:NSAccessibilityTextFieldRole] ||
[role isEqualToString:NSAccessibilityTextAreaRole]) {
index = 5; // IDC_IBEAM for text input fields
}
} else if ([role isEqualToString:NSAccessibilityLinkRole] ||
[role isEqualToString:@"AXLink"]) {
index = 3; // IDC_HAND for links
} else if ([role isEqualToString:NSAccessibilityButtonRole]) {
index = 3; // IDC_HAND for buttons (clickable)
} else if ([role isEqualToString:NSAccessibilitySplitterRole] ||
[role isEqualToString:@"AXSplitGroup"]) {
// Check orientation for resize cursor
CFTypeRef orientRef = nullptr;
if (AXUIElementCopyAttributeValue(element, CFSTR("AXOrientation"), &orientRef) == kAXErrorSuccess && orientRef) {
NSString* orient = (__bridge NSString*)orientRef;
if ([orient isEqualToString:@"AXHorizontalOrientation"]) {
index = 11; // IDC_SIZENS (vertical resize)
} else {
index = 13; // IDC_SIZEWE (horizontal resize)
}
CFRelease(orientRef);
} else {
index = 13; // IDC_SIZEWE default for splitters
}
} else if ([role isEqualToString:NSAccessibilityGrowAreaRole]) {
index = 12; // IDC_SIZENWSE for resize corners
}
CFRelease(roleRef);
}
CFRelease(element);
}
// Cache the result
cachedIndex = index;
return index;
}
void ScreenHandler::captureLoop()
{
NSLog(@"ScreenHandler CaptureLoop started (%dx%d)", m_width, m_height);