Skip to content

Commit

Permalink
feat: Make mouse coordinates DPI-aware (#293)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mykola-mokhnach
mykola-mokhnach authored Feb 9, 2025
1 parent 5e0e453 commit dc32a77
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 65 deletions.
47 changes: 35 additions & 12 deletions lib/commands/gestures.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
toMouseMoveInput,
toMouseWheelInput,
getVirtualScreenSize,
ensureDpiAwareness as _ensureDpiAwareness,
} from './winapi/user32';
import { errors } from 'appium/driver';
import B from 'bluebird';
Expand Down Expand Up @@ -288,6 +289,8 @@ export async function windowsClick (
times = 1,
interClickDelayMs = 100,
) {
await ensureDpiAwareness.bind(this)();

const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [absoluteX, absoluteY] = await toAbsoluteCoordinates.bind(this)(elementId, x, y);
let clickDownInput;
Expand All @@ -299,7 +302,7 @@ export async function windowsClick (
toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.DOWN}),
toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.UP}),
toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.CLICK}),
toMouseMoveInput({x: absoluteX, y: absoluteY}),
toMouseMoveInput(absoluteX, absoluteY),
]);
} catch (e) {
throw preprocessError(e);
Expand Down Expand Up @@ -362,16 +365,18 @@ export async function windowsScroll (
deltaY,
modifierKeys,
) {
await ensureDpiAwareness.bind(this)();

const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [absoluteX, absoluteY] = await toAbsoluteCoordinates.bind(this)(elementId, x, y);
let moveInput;
let scrollInput;
try {
moveInput = await toMouseMoveInput({x: absoluteX, y: absoluteY});
scrollInput = toMouseWheelInput({
dx: /** @type {number} */ (deltaX),
dy: /** @type {number} */ (deltaY),
});
moveInput = await toMouseMoveInput(absoluteX, absoluteY);
scrollInput = toMouseWheelInput(
/** @type {number} */ (deltaX),
/** @type {number} */ (deltaY),
);
} catch (e) {
throw preprocessError(e);
}
Expand Down Expand Up @@ -431,6 +436,8 @@ export async function windowsClickAndDrag (
modifierKeys,
durationMs = 5000,
) {
await ensureDpiAwareness.bind(this)();

const screenSize = await getVirtualScreenSize();
const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [[startAbsoluteX, startAbsoluteY], [endAbsoluteX, endAbsoluteY]] = await B.all([
Expand All @@ -443,9 +450,9 @@ export async function windowsClickAndDrag (
let moveEndInput;
try {
[moveStartInput, clickDownInput, moveEndInput, clickUpInput] = await B.all([
toMouseMoveInput({x: startAbsoluteX, y: startAbsoluteY}, screenSize),
toMouseMoveInput(startAbsoluteX, startAbsoluteY, screenSize),
toMouseButtonInput({button: MOUSE_BUTTON.LEFT, action: MOUSE_BUTTON_ACTION.DOWN}),
toMouseMoveInput({x: endAbsoluteX, y: endAbsoluteY}, screenSize),
toMouseMoveInput(endAbsoluteX, endAbsoluteY, screenSize),
toMouseButtonInput({button: MOUSE_BUTTON.LEFT, action: MOUSE_BUTTON_ACTION.UP}),
]);
} catch (e) {
Expand Down Expand Up @@ -509,6 +516,8 @@ export async function windowsHover (
modifierKeys,
durationMs = 500,
) {
await ensureDpiAwareness.bind(this)();

const screenSize = await getVirtualScreenSize();
const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys);
const [[startAbsoluteX, startAbsoluteY], [endAbsoluteX, endAbsoluteY]] = await B.all([
Expand All @@ -520,10 +529,11 @@ export async function windowsHover (
const inputPromisesChunk = [];
const maxChunkSize = 10;
for (let step = 0; step <= stepsCount; ++step) {
const promise = B.resolve(toMouseMoveInput({
x: startAbsoluteX + Math.trunc((endAbsoluteX - startAbsoluteX) * step / stepsCount),
y: startAbsoluteY + Math.trunc((endAbsoluteY - startAbsoluteY) * step / stepsCount),
}, screenSize));
const promise = B.resolve(toMouseMoveInput(
startAbsoluteX + Math.trunc((endAbsoluteX - startAbsoluteX) * step / stepsCount),
startAbsoluteY + Math.trunc((endAbsoluteY - startAbsoluteY) * step / stepsCount),
screenSize
));
inputPromises.push(promise);
// This is needed to avoid 'Error: Too many asynchronous calls are running'
inputPromisesChunk.push(promise);
Expand Down Expand Up @@ -590,6 +600,19 @@ export async function windowsKeys (actions) {
}
}

/**
* @this {WindowsDriver}
* @returns {Promise<void>}
*/
async function ensureDpiAwareness() {
if (!await _ensureDpiAwareness()) {
this.log.info(
`The call to SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) API has failed. ` +
`Mouse cursor coordinates calculation for scaled displays might not work as expected.`
);
}
}

/**
* @typedef {import('../driver').WindowsDriver} WindowsDriver
*/
115 changes: 62 additions & 53 deletions lib/commands/winapi/user32.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ const getUser32 = _.memoize(function getUser32() {
GetSystemMetrics: nodeUtil.promisify(
user32.func('int __stdcall GetSystemMetrics(int nIndex)').async
),
SetProcessDpiAwarenessContext: nodeUtil.promisify(
user32.func('int __stdcall SetProcessDpiAwarenessContext(int value)').async
)
};
});

Expand Down Expand Up @@ -158,11 +161,16 @@ const MOUSEEVENTF_VIRTUALDESK = 0x4000;
const MOUSEEVENTF_ABSOLUTE = 0x8000;
const XBUTTON1 = 0x0001;
const XBUTTON2 = 0x0002;
const SM_XVIRTUALSCREEN = 76;
const SM_YVIRTUALSCREEN = 77;
const SM_CXVIRTUALSCREEN = 78;
const SM_CYVIRTUALSCREEN = 79;
const MOUSE_MOVE_NORM = 0xFFFF;
const WHEEL_DELTA = 120;

// const DPI_AWARENESS_CONTEXT_UNAWARE = 16;
// const DPI_AWARENESS_CONTEXT_SYSTEM_AWARE = 17;
// const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE = 18;
const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = 34;

export function createKeyInput(params = {}) {
return {
Expand Down Expand Up @@ -205,6 +213,18 @@ export async function handleInputs(inputs) {
return uSent;
}

/** @type {() => Promise<boolean>} */
export const ensureDpiAwareness = _.memoize(async function ensureDpiAwareness() {
return Boolean(
await getUser32().SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)
);
});

/**
*
* @param {number} nIndex
* @returns {Promise<number>}
*/
async function getSystemMetrics(nIndex) {
return await getUser32().GetSystemMetrics(nIndex);
}
Expand Down Expand Up @@ -306,77 +326,62 @@ export async function toMouseButtonInput({button, action}) {
});
}

/**
*
* @param {number} num
* @param {number} min
* @param {number} max
* @returns {number}
*/
function clamp (num, min, max) {
return Math.min(Math.max(num, min), max);
}

/**
* @typedef {Object} MouseMoveOptions
* @property {number} dx Horizontal delta relative to the current cursor position as an integer.
* Most be provided if dy is present
* @property {number} dy Vertical delta relative to the current cursor position as an integer.
* Most be provided if dx is present
* @property {number} x Horizontal absolute cursor position on the virtual desktop as an integer.
* Most be provided if y is present
* @property {number} y Vertical absolute cursor position on the virtual desktop as an integer.
* Most be provided if x is present
*/

/**
* Transforms given mouse move parameters into an appropriate
* input structure
*
* @param {Partial<MouseMoveOptions>} opts
* @param {Size?} screenSize
* @see https://www.reddit.com/r/cpp_questions/comments/1eslzdv/difficulty_with_win32_mouse_position/
* @param {number} x Horizontal absolute cursor position on the virtual desktop as an integer.
* Most be provided if y is present
* @param {number} y Vertical absolute cursor position on the virtual desktop as an integer.
* Most be provided if x is present
* @param {import('@appium/types').Size | null} [screenSize=null]
* @returns {Promise<INPUT>} The resulting input structure
* @throws {Error} If the input data is invalid
*/
export async function toMouseMoveInput({dx, dy, x, y}, screenSize = null) {
const isAbsolute = _.isInteger(x) && _.isInteger(y);
const isRelative = _.isInteger(dx) && _.isInteger(dy);
if (!isAbsolute && !isRelative) {
throw createInvalidArgumentError('Either relative or absolute move coordinates must be provided');
export async function toMouseMoveInput(x, y, screenSize = null) {
if (!_.isInteger(x) || !_.isInteger(y)) {
throw createInvalidArgumentError('Both move coordinates must be provided');
}

if (isAbsolute) {
const {width, height} = screenSize ?? await getVirtualScreenSize();
if (width <= 1 || height <= 1) {
throw new Error('Cannot retrieve virtual screen dimensions via GetSystemMetrics WinAPI');
}
x = clamp(x, 0, width);
y = clamp(y, 0, height);
return createMouseInput({
dx: (x * MOUSE_MOVE_NORM) / (width - 1),
dy: (y * MOUSE_MOVE_NORM) / (height - 1),
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
});
const {width, height} = screenSize ?? await getVirtualScreenSize();
if (width <= 1 || height <= 1) {
throw new Error('Cannot retrieve virtual screen dimensions via GetSystemMetrics WinAPI');
}
// Relative coordinates

const {x: startX, y: startY} = await getVirtualScreenPosition();
const clampedX = clamp(/** @type {number} */ (x) - startX, 0, width);
const clampedY = clamp(/** @type {number} */ (y) - startY, 0, height);
return createMouseInput({
dx, dy,
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_VIRTUALDESK,
dx: (clampedX * MOUSE_MOVE_NORM) / (width - 1),
dy: (clampedY * MOUSE_MOVE_NORM) / (height - 1),
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
});
}

/**
* @typedef {Object} MouseWheelOptions
* @property {number} dx Horizontal scroll delta as an integer.
* If provided then no vertical scroll delta must be set.
* @property {number} dy Vertical scroll delta as an integer.
* If provided then no horizontal scroll delta must be set.
*/

/**
* Transforms given mouse wheel parameters into an appropriate
* input structure
*
* @param {MouseWheelOptions} opts
* @param {number} dx Horizontal scroll delta as an integer.
* If provided then no vertical scroll delta must be set.
* @param {number} dy Vertical scroll delta as an integer.
* If provided then no horizontal scroll delta must be set.
* @returns {INPUT | null} The resulting input structure or null
* if no input has been generated.
* @throws {Error} If the input data is invalid
*/
export function toMouseWheelInput({dx, dy}) {
export function toMouseWheelInput(dx, dy) {
const hasHorizontalScroll = _.isInteger(dx);
const hasVerticalScroll = _.isInteger(dy);
if (!hasHorizontalScroll && !hasVerticalScroll) {
Expand Down Expand Up @@ -432,18 +437,22 @@ export function toUnicodeKeyInputs(text) {
return result;
}

/**
* @typedef {Object} Size
* @property {number} width
* @property {number} height
*/

/**
* Fetches the size of the virtual screen
*
* @returns {Promise<Size>}
* @returns {Promise<import('@appium/types').Size>}
*/
export async function getVirtualScreenSize () {
const [width, height] = await B.all([SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN].map(getSystemMetrics));
return {width, height};
}

/**
* Fetches the location of the virtual screen
*
* @returns {Promise<import('@appium/types').Position>}
*/
export async function getVirtualScreenPosition () {
const [x, y] = await B.all([SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN].map(getSystemMetrics));
return {x, y};
}

0 comments on commit dc32a77

Please sign in to comment.