Feature: Add cursor position and type detection for macOS client
This commit is contained in:
@@ -93,6 +93,12 @@ private:
|
|||||||
// Get current time in milliseconds
|
// Get current time in milliseconds
|
||||||
static uint64_t getTickMs();
|
static uint64_t getTickMs();
|
||||||
|
|
||||||
|
// Get current cursor position (in physical pixels)
|
||||||
|
void getCursorPosition(int32_t& x, int32_t& y);
|
||||||
|
|
||||||
|
// Get current cursor type index (matches Windows cursor indices)
|
||||||
|
uint8_t getCursorTypeIndex();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
IOCPClient* m_client;
|
IOCPClient* m_client;
|
||||||
uint64_t m_clientID;
|
uint64_t m_clientID;
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
#import "Permissions.h"
|
#import "Permissions.h"
|
||||||
#import <Cocoa/Cocoa.h>
|
#import <Cocoa/Cocoa.h>
|
||||||
#import <CoreGraphics/CoreGraphics.h>
|
#import <CoreGraphics/CoreGraphics.h>
|
||||||
|
#import <ApplicationServices/ApplicationServices.h>
|
||||||
#import <mach/mach_time.h>
|
#import <mach/mach_time.h>
|
||||||
|
|
||||||
// Global client ID (calculated in main.mm)
|
// Global client ID (calculated in main.mm)
|
||||||
@@ -331,13 +332,14 @@ void ScreenHandler::sendDiffFrame()
|
|||||||
uint8_t algo = m_algorithm.load();
|
uint8_t algo = m_algorithm.load();
|
||||||
memcpy(data, &algo, sizeof(uint8_t));
|
memcpy(data, &algo, sizeof(uint8_t));
|
||||||
|
|
||||||
// Write cursor position (simple 0 for now)
|
// Write cursor position
|
||||||
int32_t cursorX = 0, cursorY = 0;
|
int32_t cursorX, cursorY;
|
||||||
|
getCursorPosition(cursorX, cursorY);
|
||||||
memcpy(data + 1, &cursorX, sizeof(int32_t));
|
memcpy(data + 1, &cursorX, sizeof(int32_t));
|
||||||
memcpy(data + 1 + sizeof(int32_t), &cursorY, sizeof(int32_t));
|
memcpy(data + 1 + sizeof(int32_t), &cursorY, sizeof(int32_t));
|
||||||
|
|
||||||
// Write cursor type
|
// Write cursor type
|
||||||
uint8_t cursorType = 0;
|
uint8_t cursorType = getCursorTypeIndex();
|
||||||
memcpy(data + 1 + 2 * sizeof(int32_t), &cursorType, sizeof(uint8_t));
|
memcpy(data + 1 + 2 * sizeof(int32_t), &cursorType, sizeof(uint8_t));
|
||||||
|
|
||||||
uint32_t headerSize = 1 + 2 * sizeof(int32_t) + 1;
|
uint32_t headerSize = 1 + 2 * sizeof(int32_t) + 1;
|
||||||
@@ -407,13 +409,14 @@ void ScreenHandler::sendH264Frame(bool keyframe)
|
|||||||
packet[0] = TOKEN_NEXTSCREEN;
|
packet[0] = TOKEN_NEXTSCREEN;
|
||||||
packet[1] = ALGORITHM_H264;
|
packet[1] = ALGORITHM_H264;
|
||||||
|
|
||||||
// Cursor position (0 for now)
|
// Cursor position
|
||||||
int32_t cursorX = 0, cursorY = 0;
|
int32_t cursorX, cursorY;
|
||||||
|
getCursorPosition(cursorX, cursorY);
|
||||||
memcpy(&packet[2], &cursorX, sizeof(int32_t));
|
memcpy(&packet[2], &cursorX, sizeof(int32_t));
|
||||||
memcpy(&packet[2 + sizeof(int32_t)], &cursorY, sizeof(int32_t));
|
memcpy(&packet[2 + sizeof(int32_t)], &cursorY, sizeof(int32_t));
|
||||||
|
|
||||||
// Cursor type
|
// Cursor type
|
||||||
packet[2 + 2 * sizeof(int32_t)] = 0;
|
packet[2 + 2 * sizeof(int32_t)] = getCursorTypeIndex();
|
||||||
|
|
||||||
// H264 data
|
// H264 data
|
||||||
memcpy(&packet[headerSize], encodedData, encodedSize);
|
memcpy(&packet[headerSize], encodedData, encodedSize);
|
||||||
@@ -516,6 +519,120 @@ uint64_t ScreenHandler::getTickMs()
|
|||||||
return (now * timebase.numer / timebase.denom) / 1000000;
|
return (now * timebase.numer / timebase.denom) / 1000000;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cached logical cursor position (shared between getCursorPosition and getCursorTypeIndex)
|
||||||
|
static CGPoint s_cachedLogicalPos = {0, 0};
|
||||||
|
|
||||||
|
void ScreenHandler::getCursorPosition(int32_t& x, int32_t& y)
|
||||||
|
{
|
||||||
|
// Get cursor position in logical (point) coordinates
|
||||||
|
CGEventRef event = CGEventCreate(nullptr);
|
||||||
|
s_cachedLogicalPos = CGEventGetLocation(event);
|
||||||
|
CFRelease(event);
|
||||||
|
|
||||||
|
// Convert to physical pixel coordinates (for Retina displays)
|
||||||
|
x = (int32_t)(s_cachedLogicalPos.x * m_scaleFactor);
|
||||||
|
y = (int32_t)(s_cachedLogicalPos.y * m_scaleFactor);
|
||||||
|
|
||||||
|
// Clamp to screen bounds
|
||||||
|
if (x < 0) x = 0;
|
||||||
|
if (y < 0) y = 0;
|
||||||
|
if (x >= m_width) x = m_width - 1;
|
||||||
|
if (y >= m_height) y = m_height - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint8_t ScreenHandler::getCursorTypeIndex()
|
||||||
|
{
|
||||||
|
// Windows cursor type indices (from CursorInfo.h):
|
||||||
|
// 0: IDC_APPSTARTING, 1: IDC_ARROW, 2: IDC_CROSS, 3: IDC_HAND,
|
||||||
|
// 4: IDC_HELP, 5: IDC_IBEAM, 6: IDC_ICON, 7: IDC_NO,
|
||||||
|
// 8: IDC_SIZE, 9: IDC_SIZEALL, 10: IDC_SIZENESW, 11: IDC_SIZENS,
|
||||||
|
// 12: IDC_SIZENWSE, 13: IDC_SIZEWE, 14: IDC_UPARROW, 15: IDC_WAIT
|
||||||
|
|
||||||
|
// NSCursor.currentSystemCursor doesn't work for background daemons.
|
||||||
|
// Use Accessibility API to infer cursor type from the UI element under cursor.
|
||||||
|
// Throttle to avoid performance impact (check every 100ms)
|
||||||
|
|
||||||
|
static uint8_t cachedIndex = 1;
|
||||||
|
static uint64_t lastCheckTime = 0;
|
||||||
|
static CGPoint lastPos = {-1, -1};
|
||||||
|
|
||||||
|
// Reuse cursor position from getCursorPosition (called before this)
|
||||||
|
CGPoint pos = s_cachedLogicalPos;
|
||||||
|
|
||||||
|
// Throttle: only check if cursor moved significantly or 100ms elapsed
|
||||||
|
uint64_t now = getTickMs();
|
||||||
|
bool posChanged = (fabs(pos.x - lastPos.x) > 5 || fabs(pos.y - lastPos.y) > 5);
|
||||||
|
if (!posChanged && (now - lastCheckTime) < 100) {
|
||||||
|
return cachedIndex;
|
||||||
|
}
|
||||||
|
lastCheckTime = now;
|
||||||
|
lastPos = pos;
|
||||||
|
|
||||||
|
uint8_t index = 1; // Default to arrow
|
||||||
|
|
||||||
|
// Get the UI element at cursor position using Accessibility API
|
||||||
|
AXUIElementRef systemWide = AXUIElementCreateSystemWide();
|
||||||
|
AXUIElementRef element = nullptr;
|
||||||
|
|
||||||
|
AXError err = AXUIElementCopyElementAtPosition(systemWide, (float)pos.x, (float)pos.y, &element);
|
||||||
|
CFRelease(systemWide);
|
||||||
|
|
||||||
|
if (err == kAXErrorSuccess && element) {
|
||||||
|
// Get the role of the element
|
||||||
|
CFTypeRef roleRef = nullptr;
|
||||||
|
if (AXUIElementCopyAttributeValue(element, kAXRoleAttribute, &roleRef) == kAXErrorSuccess && roleRef) {
|
||||||
|
NSString* role = (__bridge NSString*)roleRef;
|
||||||
|
|
||||||
|
// Map UI element roles to cursor types
|
||||||
|
if ([role isEqualToString:NSAccessibilityTextFieldRole] ||
|
||||||
|
[role isEqualToString:NSAccessibilityTextAreaRole] ||
|
||||||
|
[role isEqualToString:NSAccessibilityStaticTextRole] ||
|
||||||
|
[role isEqualToString:@"AXWebArea"]) {
|
||||||
|
// Check if text is editable
|
||||||
|
CFTypeRef editableRef = nullptr;
|
||||||
|
if (AXUIElementCopyAttributeValue(element, CFSTR("AXEditable"), &editableRef) == kAXErrorSuccess) {
|
||||||
|
if (editableRef && CFBooleanGetValue((CFBooleanRef)editableRef)) {
|
||||||
|
index = 5; // IDC_IBEAM for editable text
|
||||||
|
}
|
||||||
|
if (editableRef) CFRelease(editableRef);
|
||||||
|
} else if ([role isEqualToString:NSAccessibilityTextFieldRole] ||
|
||||||
|
[role isEqualToString:NSAccessibilityTextAreaRole]) {
|
||||||
|
index = 5; // IDC_IBEAM for text input fields
|
||||||
|
}
|
||||||
|
} else if ([role isEqualToString:NSAccessibilityLinkRole] ||
|
||||||
|
[role isEqualToString:@"AXLink"]) {
|
||||||
|
index = 3; // IDC_HAND for links
|
||||||
|
} else if ([role isEqualToString:NSAccessibilityButtonRole]) {
|
||||||
|
index = 3; // IDC_HAND for buttons (clickable)
|
||||||
|
} else if ([role isEqualToString:NSAccessibilitySplitterRole] ||
|
||||||
|
[role isEqualToString:@"AXSplitGroup"]) {
|
||||||
|
// Check orientation for resize cursor
|
||||||
|
CFTypeRef orientRef = nullptr;
|
||||||
|
if (AXUIElementCopyAttributeValue(element, CFSTR("AXOrientation"), &orientRef) == kAXErrorSuccess && orientRef) {
|
||||||
|
NSString* orient = (__bridge NSString*)orientRef;
|
||||||
|
if ([orient isEqualToString:@"AXHorizontalOrientation"]) {
|
||||||
|
index = 11; // IDC_SIZENS (vertical resize)
|
||||||
|
} else {
|
||||||
|
index = 13; // IDC_SIZEWE (horizontal resize)
|
||||||
|
}
|
||||||
|
CFRelease(orientRef);
|
||||||
|
} else {
|
||||||
|
index = 13; // IDC_SIZEWE default for splitters
|
||||||
|
}
|
||||||
|
} else if ([role isEqualToString:NSAccessibilityGrowAreaRole]) {
|
||||||
|
index = 12; // IDC_SIZENWSE for resize corners
|
||||||
|
}
|
||||||
|
|
||||||
|
CFRelease(roleRef);
|
||||||
|
}
|
||||||
|
CFRelease(element);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cache the result
|
||||||
|
cachedIndex = index;
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
void ScreenHandler::captureLoop()
|
void ScreenHandler::captureLoop()
|
||||||
{
|
{
|
||||||
NSLog(@"ScreenHandler CaptureLoop started (%dx%d)", m_width, m_height);
|
NSLog(@"ScreenHandler CaptureLoop started (%dx%d)", m_width, m_height);
|
||||||
|
|||||||
Reference in New Issue
Block a user