diff --git a/lib/commands/gestures.js b/lib/commands/gestures.js index d19cac2..bc36dba 100644 --- a/lib/commands/gestures.js +++ b/lib/commands/gestures.js @@ -12,6 +12,7 @@ import { toMouseMoveInput, toMouseWheelInput, getVirtualScreenSize, + ensureDpiAwareness as _ensureDpiAwareness, } from './winapi/user32'; import { errors } from 'appium/driver'; import B from 'bluebird'; @@ -288,6 +289,8 @@ export async function windowsClick ( times = 1, interClickDelayMs = 100, ) { + await ensureDpiAwareness.bind(this)(); + const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys); const [absoluteX, absoluteY] = await toAbsoluteCoordinates.bind(this)(elementId, x, y); let clickDownInput; @@ -299,7 +302,7 @@ export async function windowsClick ( toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.DOWN}), toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.UP}), toMouseButtonInput({button, action: MOUSE_BUTTON_ACTION.CLICK}), - toMouseMoveInput({x: absoluteX, y: absoluteY}), + toMouseMoveInput(absoluteX, absoluteY), ]); } catch (e) { throw preprocessError(e); @@ -362,16 +365,18 @@ export async function windowsScroll ( deltaY, modifierKeys, ) { + await ensureDpiAwareness.bind(this)(); + const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys); const [absoluteX, absoluteY] = await toAbsoluteCoordinates.bind(this)(elementId, x, y); let moveInput; let scrollInput; try { - moveInput = await toMouseMoveInput({x: absoluteX, y: absoluteY}); - scrollInput = toMouseWheelInput({ - dx: /** @type {number} */ (deltaX), - dy: /** @type {number} */ (deltaY), - }); + moveInput = await toMouseMoveInput(absoluteX, absoluteY); + scrollInput = toMouseWheelInput( + /** @type {number} */ (deltaX), + /** @type {number} */ (deltaY), + ); } catch (e) { throw preprocessError(e); } @@ -431,6 +436,8 @@ export async function windowsClickAndDrag ( modifierKeys, durationMs = 5000, ) { + await ensureDpiAwareness.bind(this)(); + const screenSize = await getVirtualScreenSize(); const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys); const [[startAbsoluteX, startAbsoluteY], [endAbsoluteX, endAbsoluteY]] = await B.all([ @@ -443,9 +450,9 @@ export async function windowsClickAndDrag ( let moveEndInput; try { [moveStartInput, clickDownInput, moveEndInput, clickUpInput] = await B.all([ - toMouseMoveInput({x: startAbsoluteX, y: startAbsoluteY}, screenSize), + toMouseMoveInput(startAbsoluteX, startAbsoluteY, screenSize), toMouseButtonInput({button: MOUSE_BUTTON.LEFT, action: MOUSE_BUTTON_ACTION.DOWN}), - toMouseMoveInput({x: endAbsoluteX, y: endAbsoluteY}, screenSize), + toMouseMoveInput(endAbsoluteX, endAbsoluteY, screenSize), toMouseButtonInput({button: MOUSE_BUTTON.LEFT, action: MOUSE_BUTTON_ACTION.UP}), ]); } catch (e) { @@ -509,6 +516,8 @@ export async function windowsHover ( modifierKeys, durationMs = 500, ) { + await ensureDpiAwareness.bind(this)(); + const screenSize = await getVirtualScreenSize(); const [modifierKeyDownInputs, modifierKeyUpInputs] = modifierKeysToInputs.bind(this)(modifierKeys); const [[startAbsoluteX, startAbsoluteY], [endAbsoluteX, endAbsoluteY]] = await B.all([ @@ -520,10 +529,11 @@ export async function windowsHover ( const inputPromisesChunk = []; const maxChunkSize = 10; for (let step = 0; step <= stepsCount; ++step) { - const promise = B.resolve(toMouseMoveInput({ - x: startAbsoluteX + Math.trunc((endAbsoluteX - startAbsoluteX) * step / stepsCount), - y: startAbsoluteY + Math.trunc((endAbsoluteY - startAbsoluteY) * step / stepsCount), - }, screenSize)); + const promise = B.resolve(toMouseMoveInput( + startAbsoluteX + Math.trunc((endAbsoluteX - startAbsoluteX) * step / stepsCount), + startAbsoluteY + Math.trunc((endAbsoluteY - startAbsoluteY) * step / stepsCount), + screenSize + )); inputPromises.push(promise); // This is needed to avoid 'Error: Too many asynchronous calls are running' inputPromisesChunk.push(promise); @@ -590,6 +600,19 @@ export async function windowsKeys (actions) { } } +/** + * @this {WindowsDriver} + * @returns {Promise} + */ +async function ensureDpiAwareness() { + if (!await _ensureDpiAwareness()) { + this.log.info( + `The call to SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) API has failed. ` + + `Mouse cursor coordinates calculation for scaled displays might not work as expected.` + ); + } +} + /** * @typedef {import('../driver').WindowsDriver} WindowsDriver */ diff --git a/lib/commands/winapi/user32.js b/lib/commands/winapi/user32.js index 64b6ce3..bc37e62 100644 --- a/lib/commands/winapi/user32.js +++ b/lib/commands/winapi/user32.js @@ -41,6 +41,9 @@ const getUser32 = _.memoize(function getUser32() { GetSystemMetrics: nodeUtil.promisify( user32.func('int __stdcall GetSystemMetrics(int nIndex)').async ), + SetProcessDpiAwarenessContext: nodeUtil.promisify( + user32.func('int __stdcall SetProcessDpiAwarenessContext(int value)').async + ) }; }); @@ -158,11 +161,16 @@ const MOUSEEVENTF_VIRTUALDESK = 0x4000; const MOUSEEVENTF_ABSOLUTE = 0x8000; const XBUTTON1 = 0x0001; const XBUTTON2 = 0x0002; +const SM_XVIRTUALSCREEN = 76; +const SM_YVIRTUALSCREEN = 77; const SM_CXVIRTUALSCREEN = 78; const SM_CYVIRTUALSCREEN = 79; const MOUSE_MOVE_NORM = 0xFFFF; const WHEEL_DELTA = 120; - +// const DPI_AWARENESS_CONTEXT_UNAWARE = 16; +// const DPI_AWARENESS_CONTEXT_SYSTEM_AWARE = 17; +// const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE = 18; +const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = 34; export function createKeyInput(params = {}) { return { @@ -205,6 +213,18 @@ export async function handleInputs(inputs) { return uSent; } +/** @type {() => Promise} */ +export const ensureDpiAwareness = _.memoize(async function ensureDpiAwareness() { + return Boolean( + await getUser32().SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) + ); +}); + +/** + * + * @param {number} nIndex + * @returns {Promise} + */ async function getSystemMetrics(nIndex) { return await getUser32().GetSystemMetrics(nIndex); } @@ -306,77 +326,62 @@ export async function toMouseButtonInput({button, action}) { }); } +/** + * + * @param {number} num + * @param {number} min + * @param {number} max + * @returns {number} + */ function clamp (num, min, max) { return Math.min(Math.max(num, min), max); } -/** - * @typedef {Object} MouseMoveOptions - * @property {number} dx Horizontal delta relative to the current cursor position as an integer. - * Most be provided if dy is present - * @property {number} dy Vertical delta relative to the current cursor position as an integer. - * Most be provided if dx is present - * @property {number} x Horizontal absolute cursor position on the virtual desktop as an integer. - * Most be provided if y is present - * @property {number} y Vertical absolute cursor position on the virtual desktop as an integer. - * Most be provided if x is present - */ - /** * Transforms given mouse move parameters into an appropriate * input structure * - * @param {Partial} opts - * @param {Size?} screenSize + * @see https://www.reddit.com/r/cpp_questions/comments/1eslzdv/difficulty_with_win32_mouse_position/ + * @param {number} x Horizontal absolute cursor position on the virtual desktop as an integer. + * Most be provided if y is present + * @param {number} y Vertical absolute cursor position on the virtual desktop as an integer. + * Most be provided if x is present + * @param {import('@appium/types').Size | null} [screenSize=null] * @returns {Promise} The resulting input structure * @throws {Error} If the input data is invalid */ -export async function toMouseMoveInput({dx, dy, x, y}, screenSize = null) { - const isAbsolute = _.isInteger(x) && _.isInteger(y); - const isRelative = _.isInteger(dx) && _.isInteger(dy); - if (!isAbsolute && !isRelative) { - throw createInvalidArgumentError('Either relative or absolute move coordinates must be provided'); +export async function toMouseMoveInput(x, y, screenSize = null) { + if (!_.isInteger(x) || !_.isInteger(y)) { + throw createInvalidArgumentError('Both move coordinates must be provided'); } - if (isAbsolute) { - const {width, height} = screenSize ?? await getVirtualScreenSize(); - if (width <= 1 || height <= 1) { - throw new Error('Cannot retrieve virtual screen dimensions via GetSystemMetrics WinAPI'); - } - x = clamp(x, 0, width); - y = clamp(y, 0, height); - return createMouseInput({ - dx: (x * MOUSE_MOVE_NORM) / (width - 1), - dy: (y * MOUSE_MOVE_NORM) / (height - 1), - dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, - }); + const {width, height} = screenSize ?? await getVirtualScreenSize(); + if (width <= 1 || height <= 1) { + throw new Error('Cannot retrieve virtual screen dimensions via GetSystemMetrics WinAPI'); } - // Relative coordinates - + const {x: startX, y: startY} = await getVirtualScreenPosition(); + const clampedX = clamp(/** @type {number} */ (x) - startX, 0, width); + const clampedY = clamp(/** @type {number} */ (y) - startY, 0, height); return createMouseInput({ - dx, dy, - dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_VIRTUALDESK, + dx: (clampedX * MOUSE_MOVE_NORM) / (width - 1), + dy: (clampedY * MOUSE_MOVE_NORM) / (height - 1), + dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, }); } -/** - * @typedef {Object} MouseWheelOptions - * @property {number} dx Horizontal scroll delta as an integer. - * If provided then no vertical scroll delta must be set. - * @property {number} dy Vertical scroll delta as an integer. - * If provided then no horizontal scroll delta must be set. - */ - /** * Transforms given mouse wheel parameters into an appropriate * input structure * - * @param {MouseWheelOptions} opts + * @param {number} dx Horizontal scroll delta as an integer. + * If provided then no vertical scroll delta must be set. + * @param {number} dy Vertical scroll delta as an integer. + * If provided then no horizontal scroll delta must be set. * @returns {INPUT | null} The resulting input structure or null * if no input has been generated. * @throws {Error} If the input data is invalid */ -export function toMouseWheelInput({dx, dy}) { +export function toMouseWheelInput(dx, dy) { const hasHorizontalScroll = _.isInteger(dx); const hasVerticalScroll = _.isInteger(dy); if (!hasHorizontalScroll && !hasVerticalScroll) { @@ -432,18 +437,22 @@ export function toUnicodeKeyInputs(text) { return result; } -/** - * @typedef {Object} Size - * @property {number} width - * @property {number} height - */ - /** * Fetches the size of the virtual screen * - * @returns {Promise} + * @returns {Promise} */ export async function getVirtualScreenSize () { const [width, height] = await B.all([SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN].map(getSystemMetrics)); return {width, height}; } + +/** + * Fetches the location of the virtual screen + * + * @returns {Promise} + */ +export async function getVirtualScreenPosition () { + const [x, y] = await B.all([SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN].map(getSystemMetrics)); + return {x, y}; +}