Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Make mouse coordinates DPI-aware #293

Merged
merged 4 commits into from
Feb 9, 2025
Merged

feat: Make mouse coordinates DPI-aware #293

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
926b1fb
feat: Make mouse coordinates DPI-aware
mykola-mokhnach Feb 8, 2025
b0190b3
line breaks
mykola-mokhnach Feb 8, 2025
4a6f703
Address comments
mykola-mokhnach Feb 9, 2025
da38923
moar
mykola-mokhnach Feb 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 22 additions & 12 deletions lib/commands/gestures.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
toMouseMoveInput,
toMouseWheelInput,
getVirtualScreenSize,
ensureDpiAwareness,
} from './winapi/user32';
import { errors } from 'appium/driver';
import B from 'bluebird';
Expand Down Expand Up @@ -288,6 +289,8 @@ export async function windowsClick (
times = 1,
interClickDelayMs = 100,
) {
await ensureDpiAwareness();

const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [absoluteX, absoluteY] = await toAbsoluteCoordinates.bind(this)(elementId, x, y);
let clickDownInput;
Expand All @@ -299,7 +302,7 @@ export async function windowsClick (
toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.DOWN}),
toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.UP}),
toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.CLICK}),
toMouseMoveInput({x: absoluteX, y: absoluteY}),
toMouseMoveInput(absoluteX, absoluteY),
]);
} catch (e) {
throw preprocessError(e);
Expand Down Expand Up @@ -362,16 +365,18 @@ export async function windowsScroll (
deltaY,
modifierKeys,
) {
await ensureDpiAwareness();

const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [absoluteX, absoluteY] = await toAbsoluteCoordinates.bind(this)(elementId, x, y);
let moveInput;
let scrollInput;
try {
moveInput = await toMouseMoveInput({x: absoluteX, y: absoluteY});
scrollInput = toMouseWheelInput({
dx: /** @type {number} */ (deltaX),
dy: /** @type {number} */ (deltaY),
});
moveInput = await toMouseMoveInput(absoluteX, absoluteY);
scrollInput = toMouseWheelInput(
/** @type {number} */ (deltaX),
/** @type {number} */ (deltaY),
);
} catch (e) {
throw preprocessError(e);
}
Expand Down Expand Up @@ -431,6 +436,8 @@ export async function windowsClickAndDrag (
modifierKeys,
durationMs = 5000,
) {
await ensureDpiAwareness();

const screenSize = await getVirtualScreenSize();
const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [[startAbsoluteX, startAbsoluteY], [endAbsoluteX, endAbsoluteY]] = await B.all([
Expand All @@ -443,9 +450,9 @@ export async function windowsClickAndDrag (
let moveEndInput;
try {
[moveStartInput, clickDownInput, moveEndInput, clickUpInput] = await B.all([
toMouseMoveInput({x: startAbsoluteX, y: startAbsoluteY}, screenSize),
toMouseMoveInput(startAbsoluteX, startAbsoluteY, screenSize),
toMouseButtonInput({button: MOUSE_BUTTON.LEFT, action: MOUSE_BUTTON_ACTION.DOWN}),
toMouseMoveInput({x: endAbsoluteX, y: endAbsoluteY}, screenSize),
toMouseMoveInput(endAbsoluteX, endAbsoluteY, screenSize),
toMouseButtonInput({button: MOUSE_BUTTON.LEFT, action: MOUSE_BUTTON_ACTION.UP}),
]);
} catch (e) {
Expand Down Expand Up @@ -509,6 +516,8 @@ export async function windowsHover (
modifierKeys,
durationMs = 500,
) {
await ensureDpiAwareness();

const screenSize = await getVirtualScreenSize();
const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [[startAbsoluteX, startAbsoluteY], [endAbsoluteX, endAbsoluteY]] = await B.all([
Expand All @@ -520,10 +529,11 @@ export async function windowsHover (
const inputPromisesChunk = [];
const maxChunkSize = 10;
for (let step = 0; step <= stepsCount; ++step) {
const promise = B.resolve(toMouseMoveInput({
x: startAbsoluteX + Math.trunc((endAbsoluteX - startAbsoluteX) * step / stepsCount),
y: startAbsoluteY + Math.trunc((endAbsoluteY - startAbsoluteY) * step / stepsCount),
}, screenSize));
const promise = B.resolve(toMouseMoveInput(
startAbsoluteX + Math.trunc((endAbsoluteX - startAbsoluteX) * step / stepsCount),
startAbsoluteY + Math.trunc((endAbsoluteY - startAbsoluteY) * step / stepsCount),
screenSize
));
inputPromises.push(promise);
// This is needed to avoid 'Error: Too many asynchronous calls are running'
inputPromisesChunk.push(promise);
Expand Down
112 changes: 59 additions & 53 deletions lib/commands/winapi/user32.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ const getUser32 = _.memoize(function getUser32() {
GetSystemMetrics: nodeUtil.promisify(
user32.func('int __stdcall GetSystemMetrics(int nIndex)').async
),
SetProcessDpiAwarenessContext: nodeUtil.promisify(
user32.func('int __stdcall SetProcessDpiAwarenessContext(int value)').async
)
};
});

Expand Down Expand Up @@ -158,11 +161,16 @@ const MOUSEEVENTF_VIRTUALDESK = 0x4000;
const MOUSEEVENTF_ABSOLUTE = 0x8000;
const XBUTTON1 = 0x0001;
const XBUTTON2 = 0x0002;
const SM_XVIRTUALSCREEN = 76;
const SM_YVIRTUALSCREEN = 77;
const SM_CXVIRTUALSCREEN = 78;
const SM_CYVIRTUALSCREEN = 79;
const MOUSE_MOVE_NORM = 0xFFFF;
const WHEEL_DELTA = 120;

// const DPI_AWARENESS_CONTEXT_UNAWARE = 16;
// const DPI_AWARENESS_CONTEXT_SYSTEM_AWARE = 17;
// const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE = 18;
const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = 34;

export function createKeyInput(params = {}) {
return {
Expand Down Expand Up @@ -205,6 +213,15 @@ export async function handleInputs(inputs) {
return uSent;
}

export const ensureDpiAwareness = _.memoize(async function ensureDpiAwareness() {
await getUser32().SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);
KazuCocoa marked this conversation as resolved.
Show resolved Hide resolved
});

/**
*
* @param {number} nIndex
* @returns {Promise<number>}
*/
async function getSystemMetrics(nIndex) {
return await getUser32().GetSystemMetrics(nIndex);
}
Expand Down Expand Up @@ -306,77 +323,62 @@ export async function toMouseButtonInput({button, action}) {
});
}

/**
*
* @param {number} num
* @param {number} min
* @param {number} max
* @returns {number}
*/
function clamp (num, min, max) {
return Math.min(Math.max(num, min), max);
}

/**
* @typedef {Object} MouseMoveOptions
* @property {number} dx Horizontal delta relative to the current cursor position as an integer.
* Most be provided if dy is present
* @property {number} dy Vertical delta relative to the current cursor position as an integer.
* Most be provided if dx is present
* @property {number} x Horizontal absolute cursor position on the virtual desktop as an integer.
* Most be provided if y is present
* @property {number} y Vertical absolute cursor position on the virtual desktop as an integer.
* Most be provided if x is present
*/

/**
* Transforms given mouse move parameters into an appropriate
* input structure
*
* @param {Partial<MouseMoveOptions>} opts
* @param {Size?} screenSize
* @see https://www.reddit.com/r/cpp_questions/comments/1eslzdv/difficulty_with_win32_mouse_position/
* @param {number} x Horizontal absolute cursor position on the virtual desktop as an integer.
* Most be provided if y is present
* @param {number} y Vertical absolute cursor position on the virtual desktop as an integer.
* Most be provided if x is present
* @param {import('@appium/types').Size | null} [screenSize=null]
* @returns {Promise<INPUT>} The resulting input structure
* @throws {Error} If the input data is invalid
*/
export async function toMouseMoveInput({dx, dy, x, y}, screenSize = null) {
const isAbsolute = _.isInteger(x) && _.isInteger(y);
const isRelative = _.isInteger(dx) && _.isInteger(dy);
if (!isAbsolute && !isRelative) {
throw createInvalidArgumentError('Either relative or absolute move coordinates must be provided');
export async function toMouseMoveInput(x, y, screenSize = null) {
if (!_.isInteger(x) || !_.isInteger(y)) {
throw createInvalidArgumentError('Both move coordinates must be provided');
}

if (isAbsolute) {
const {width, height} = screenSize ?? await getVirtualScreenSize();
if (width <= 1 || height <= 1) {
throw new Error('Cannot retrieve virtual screen dimensions via GetSystemMetrics WinAPI');
}
x = clamp(x, 0, width);
y = clamp(y, 0, height);
return createMouseInput({
dx: (x * MOUSE_MOVE_NORM) / (width - 1),
dy: (y * MOUSE_MOVE_NORM) / (height - 1),
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
});
const {width, height} = screenSize ?? await getVirtualScreenSize();
if (width <= 1 || height <= 1) {
throw new Error('Cannot retrieve virtual screen dimensions via GetSystemMetrics WinAPI');
}
// Relative coordinates

const {x: startX, y: startY} = await getVirtualScreenPosition();
const clampedX = clamp(/** @type {number} */ (x) - startX, 0, width);
const clampedY = clamp(/** @type {number} */ (y) - startY, 0, height);
return createMouseInput({
dx, dy,
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_VIRTUALDESK,
dx: (clampedX * MOUSE_MOVE_NORM) / (width - 1),
dy: (clampedY * MOUSE_MOVE_NORM) / (height - 1),
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
});
}

/**
* @typedef {Object} MouseWheelOptions
* @property {number} dx Horizontal scroll delta as an integer.
* If provided then no vertical scroll delta must be set.
* @property {number} dy Vertical scroll delta as an integer.
* If provided then no horizontal scroll delta must be set.
*/

/**
* Transforms given mouse wheel parameters into an appropriate
* input structure
*
* @param {MouseWheelOptions} opts
* @param {number} dx Horizontal scroll delta as an integer.
* If provided then no vertical scroll delta must be set.
* @param {number} dy Vertical scroll delta as an integer.
* If provided then no horizontal scroll delta must be set.
* @returns {INPUT | null} The resulting input structure or null
* if no input has been generated.
* @throws {Error} If the input data is invalid
*/
export function toMouseWheelInput({dx, dy}) {
export function toMouseWheelInput(dx, dy) {
const hasHorizontalScroll = _.isInteger(dx);
const hasVerticalScroll = _.isInteger(dy);
if (!hasHorizontalScroll && !hasVerticalScroll) {
Expand Down Expand Up @@ -432,18 +434,22 @@ export function toUnicodeKeyInputs(text) {
return result;
}

/**
* @typedef {Object} Size
* @property {number} width
* @property {number} height
*/

/**
* Fetches the size of the virtual screen
*
* @returns {Promise<Size>}
* @returns {Promise<import('@appium/types').Size>}
*/
export async function getVirtualScreenSize () {
const [width, height] = await B.all([SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN].map(getSystemMetrics));
return {width, height};
}

/**
* Fetches the location of the virtual screen
*
* @returns {Promise<import('@appium/types').Position>}
*/
export async function getVirtualScreenPosition () {
const [x, y] = await B.all([SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN].map(getSystemMetrics));
return {x, y};
}