Source code for pywinauto_recorder.recorder

"""This module contains functions and classes allowing to record a sequence of user actions.
All these functions and classes are private except the Recorder class.
"""

import sys
import os
import traceback
import time
import win32api
import win32con
from threading import Thread
import pywinauto
import overlay_arrows_and_more as oaam
import keyboard
import mouse
from collections import namedtuple
import pyperclip
import codecs
from .core import path_separator, type_separator, Strategy, is_int, \
                    get_wrapper_path, get_entry_list, get_entry, get_sorted_region, \
                    read_config_file, set_native_window_handle
from .core import find_elements as not_ttl_cached_find_elements
#from .core import find_elements
from .player import playback
from cachetools import func


@func.ttl_cache(ttl=10)
def find_elements(full_element_path=None):
	return not_ttl_cached_find_elements(full_element_path=full_element_path)


__all__ = ['Recorder']

ElementEvent = namedtuple('ElementEvent', ['strategy', 'rectangle', 'path'])
SendKeysEvent = namedtuple('SendKeysEvent', ['line'])
MouseWheelEvent = namedtuple('MouseWheelEvent', ['delta'])
DragAndDropEvent = namedtuple('DragAndDropEvent', ['path', 'dx1', 'dy1', 'path2', 'dx2', 'dy2'])
ClickEvent = namedtuple('ClickEvent', ['button', 'click_count', 'path', 'dx', 'dy', 'time'])
FindEvent = namedtuple('FindEvent', ['path', 'dx', 'dy', 'time'])
MenuEvent = namedtuple('MenuEvent', ['path', 'menu_path'])


class IconSet:
	""" It loads the icons from the Icons folder and stores them in the class."""
	if "__compiled__" in globals():
		path_icons = os.path.dirname(os.path.realpath(__file__)) + r'\..'
	else:
		path_icons = os.path.dirname(os.path.realpath(__file__))
	hicon_clipboard = oaam.load_ico(path_icons + r'\Icons\paste.ico', 48, 48)
	hicon_light_on = oaam.load_ico(path_icons + r'\Icons\light-on.ico', 48, 48)
	hicon_record = oaam.load_ico(path_icons + r'\Icons\record.ico', 48, 48)
	hicon_play = oaam.load_ico(path_icons + r'\Icons\play.ico', 48, 48)
	hicon_stop = oaam.load_ico(path_icons + r'\Icons\stop.ico', 48, 48)
	hicon_search = oaam.load_ico(path_icons + r'\Icons\search.ico', 48, 48)
	hicon_power = oaam.load_ico(path_icons + r'\Icons\power.ico', 48, 48)


def _escape_special_char(string):
	"""
		Is called on all paths to remove all special characters but it's not good.
		Should be moved in core
		Should be called only in get_wrapper_path and unescape_special_char in get_entry
		and in script += 'menu_click... in menu_click as it's already done
		should replace -> by _>
	"""
	for r in (("\\", "\\\\"), ("\t", "\\t"), ("\n", "\\n"), ("\r", "\\r"), ("\v", "\\v"), ("\f", "\\f"), ('"', '\\"')):
		string = string.replace(*r)
	return string


def _compute_dx_dy(x, y, rectangle):
	cx, cy = rectangle.mid_point()
	dx, dy = float(x - cx) / (rectangle.width() / 2 - 1), float(y - cy) / (rectangle.height() / 2 - 1)
	return (dx, dy)


def _write_in_file(events, relative_coordinate_mode=False):
	from pathlib import Path
	home_dir = Path.home() / 'Pywinauto recorder'
	home_dir.mkdir(parents=True, exist_ok=True)
	record_file_name = home_dir / Path('recorded ' + time.asctime().replace(':', '_') + '.py')
	print('Recording in file: ' + str(record_file_name.absolute()))
	script = "# encoding: {}\n\n".format(sys.getdefaultencoding())
	script += "from pywinauto_recorder.player import *\n\n"
	common_path = ''
	common_window = ''
	common_region = ''
	i = 0
	while i < len(events):
		e_i = events[i]
		if type(e_i) in (DragAndDropEvent, ClickEvent, FindEvent, MenuEvent):
			if e_i.path != common_path:
				new_common_path = _find_new_common_path_in_next_user_events(events, i)
				if new_common_path != common_path:
					common_path = new_common_path
					entry_list = get_entry_list(common_path)
					e_i_window = entry_list[0]
					if e_i_window != common_window:
						common_window = e_i_window
						script += '\nwith UIPath(u"' + _escape_special_char(common_window) + '"):\n'
					e_i_region = path_separator.join(entry_list[1:])
					if e_i_region != common_region and e_i_region:
						common_region = e_i_region
						script += '\twith UIPath(u"' + _escape_special_char(common_region) + '"):\n'
					else:
						common_region = ''
		if type(e_i) in (SendKeysEvent, MouseWheelEvent, DragAndDropEvent, ClickEvent, FindEvent, MenuEvent):
			if common_window:
				script += '\t'
				if common_region:
					script += '\t'
			if isinstance(e_i, SendKeysEvent):
				script += 'send_keys(' + e_i.line + ')\n'
			elif isinstance(e_i, MouseWheelEvent):
				script += 'mouse_wheel(' + str(e_i.delta) + ')\n'
			elif isinstance(e_i, DragAndDropEvent):
				p1, p2 = e_i.path, e_i.path2
				dx1, dy1 = "{:.2f}".format(round(e_i.dx1 * 100, 2)), "{:.2f}".format(round(e_i.dy1 * 100, 2))
				dx2, dy2 = "{:.2f}".format(round(e_i.dx2 * 100, 2)), "{:.2f}".format(round(e_i.dy2 * 100, 2))
				if common_path:
					p1 = _get_relative_path(common_path, p1)
					p2 = _get_relative_path(common_path, p2)
				script += 'drag_and_drop(u"' + _escape_special_char(p1)
				if relative_coordinate_mode and eval(dx1) != 0 and eval(dy1) != 0:
					script += '%(' + dx1 + ',' + dy1 + ')'
				script += '", u"' + _escape_special_char(p2)
				if relative_coordinate_mode and eval(dx2) != 0 and eval(dy2) != 0:
					script += '%(' + dx2 + ',' + dy2 + ')'
				script += '")\n'
			elif isinstance(e_i, ClickEvent):
				p = e_i.path
				dx, dy = "{:.2f}".format(round(e_i.dx * 100, 2)), "{:.2f}".format(round(e_i.dy * 100, 2))
				if common_path:
					p = _get_relative_path(common_path, p)
				str_c = ['', '', 'double_', 'triple_']
				if e_i.button == 'left':
					if e_i.count == 1:
						script += 'click(u"' + _escape_special_char(p)
					else:
						script += str_c[e_i.click_count] + 'click(u"' + _escape_special_char(p)
				else:
					script += str_c[e_i.click_count] + e_i.button + '_click(u"' + _escape_special_char(p)
				if relative_coordinate_mode and eval(dx) != 0 and eval(dy) != 0:
					script += '%(' + dx + ',' + dy + ')'
				script += '")\n'
			elif isinstance(e_i, FindEvent):
				p = e_i.path
				dx, dy = "{:.2f}".format(round(e_i.dx * 100, 2)), "{:.2f}".format(round(e_i.dy * 100, 2))
				if common_path:
					p = _get_relative_path(common_path, p)
				script += 'wrapper = find(u"' + _escape_special_char(p)
				if relative_coordinate_mode and eval(dx) != 0 and eval(dy) != 0:
					script += '%(' + dx + ',' + dy + ')'
				script += '")\n'
			elif isinstance(e_i, MenuEvent):
				script += 'menu_click(u"' + _escape_special_char(e_i.menu_path) + '")\n'
		i += 1
	with codecs.open(record_file_name, "w", encoding=sys.getdefaultencoding()) as f:
		f.write(script)
	pyperclip.copy(script)
	return record_file_name


def _clean_events(events, remove_first_up=False):
	""""
	removes duplicate or useless events
	removes all the last down events due or not to (CRTL+ALT+r) when ending record mode
	:param remove_first_up: when True, removes the 2 first up events due to (CRTL+ALT+r) when starting record mode
	:param events: the copy of recorded event list
	"""
	if remove_first_up:
		i = 0
		i_up = 0
		while i < len(events):
			if isinstance(events[i], keyboard.KeyboardEvent) and events[i].event_type == 'up':
				i_up = i_up + 1
				events.pop(i)
				if i_up == 2:
					break
			else:
				i = i + 1
	i = 0
	previous_event_type = None
	while i < len(events):
		if type(events[i]) is previous_event_type:
			if type(events[i]) in (ElementEvent, mouse.MoveEvent):
				del events[i - 1]
			else:
				previous_event_type = type(events[i])
				i = i + 1
		else:
			previous_event_type = type(events[i])
			i = i + 1
	i = len(events) - 1
	while i > 0:
		if isinstance(events[i], keyboard.KeyboardEvent) and events[i].event_type == 'down':
			events.pop(i)
		elif isinstance(events[i], keyboard.KeyboardEvent) and events[i].event_type == 'up':
			break
		i = i - 1


def _process_events(events, process_menu_click=True):
	i = 0
	while i < len(events):
		if isinstance(events[i], keyboard.KeyboardEvent):
			_process_keyboard_events(events, i)
		elif isinstance(events[i], mouse.WheelEvent):
			_process_wheel_events(events, i)
		i = i + 1
	i = len(events) - 1
	while i >= 0:
		if isinstance(events[i], mouse.ButtonEvent) and events[i].event_type == 'up':
			i = _process_drag_and_drop_or_click_events(events, i)
		i = i - 1
	if process_menu_click:
		i = len(events) - 1
		while i >= 0:
			if isinstance(events[i], ClickEvent):
				i = _process_menu_select_events(events, i)
			i = i - 1


def _process_keyboard_events(events, i):
	keyboard_events = [events[i]]
	i0 = i + 1
	i_processed_events = []
	while i0 < len(events):
		if isinstance(events[i0], keyboard.KeyboardEvent):
			keyboard_events.append(events[i0])
			i_processed_events.append(i0)
			i0 = i0 + 1
		elif isinstance(events[i0], ElementEvent):
			i0 = i0 + 1
		else:
			break
	line = _get_send_keys_strings(keyboard_events)
	for i_p_e in sorted(i_processed_events, reverse=True):
		del events[i_p_e]
	if line:
		events[i] = SendKeysEvent(line=line)


def _process_wheel_events(events, i):
	delta = events[i].delta
	i_processed_events = []
	i0 = i + 1
	while i0 < len(events):
		if isinstance(events[i0], mouse.WheelEvent):
			delta = delta + events[i0].delta
			i_processed_events.append(i0)
			i0 = i0 + 1
		elif type(events[i0]) in (ElementEvent, mouse.MoveEvent):
			i0 = i0 + 1
		else:
			break
	for i_p_e in sorted(i_processed_events, reverse=True):
		del events[i_p_e]
	events[i] = MouseWheelEvent(delta=delta)


def _process_drag_and_drop_or_click_events(events, i):
	i0 = i - 1
	while i0 >= 0:
		if isinstance(events[i0], ElementEvent):
			element_event_before_button_up = events[i0]
			break
		i0 = i0 - 1
	while i0 >= 0:
		if isinstance(events[i0], mouse.MoveEvent):
			move_event_end = events[i0]
			break
		i0 = i0 - 1
	i0 = i - 1
	drag_and_drop = False
	click_count = 0
	while i0 >= 0:
		if isinstance(events[i0], mouse.MoveEvent):
			if events[i0].x != move_event_end.x or events[i0].y != move_event_end.y:
				drag_and_drop = True
		elif isinstance(events[i0], mouse.ButtonEvent) and events[i0].event_type in ('down', 'double'):
			click_count = click_count + 1
			if events[i0].event_type == 'down' or click_count == 3:
				i1 = i0
				break
		i0 = i0 - 1
	element_event_before_button_down = None
	while i0 >= 0:
		if isinstance(events[i0], ElementEvent):
			element_event_before_button_down = events[i0]
			break
		i0 = i0 - 1
	if drag_and_drop:
		move_event_start = None
		while i0 >= 0:
			if isinstance(events[i0], mouse.MoveEvent):
				move_event_start = events[i0]
				break
			i0 = i0 - 1
		dx1, dy1 = _compute_dx_dy(move_event_start.x, move_event_start.y, element_event_before_button_down.rectangle)
		dx2, dy2 = _compute_dx_dy(move_event_end.x, move_event_end.y, element_event_before_button_up.rectangle)
		events[i] = DragAndDropEvent(
			path=element_event_before_button_down.path, dx1=dx1, dy1=dy1,
			path2=element_event_before_button_up.path, dx2=dx2, dy2=dy2)
	else:
		up_event = events[i]
		dx, dy = _compute_dx_dy(move_event_end.x, move_event_end.y, element_event_before_button_down.rectangle)
		events[i] = ClickEvent(
			button=up_event.button, click_count=click_count,
			path=element_event_before_button_down.path, dx=dx, dy=dy, time=up_event.time)
	i_processed_events = []
	i0 = i - 1
	while i0 >= i1:
		if type(events[i0]) in (mouse.ButtonEvent, mouse.MoveEvent, ElementEvent):
			i_processed_events.append(i0)
		i0 = i0 - 1
	for i_p_e in sorted(i_processed_events, reverse=True):
		del events[i_p_e]
		i = i - 1
	return i


def _get_relative_path(common_path, path):
	if not path:
		return ''
	# TODO: check if common_path is the beginning of path
	path = path[len(common_path) + len(path_separator):]
	entry_list = get_entry_list(path)
	str_name, str_type, y_x, dx_dy = get_entry(entry_list[-1])
	if (y_x is not None) and not is_int(y_x[0]):
		y_x[0] = y_x[0][len(common_path) + 2:]
		path = path_separator.join(entry_list[:-1]) + path_separator + str_name
		if path == path_separator:
			path = ''
		path = path + type_separator + str_type + "#[" + y_x[0] + "," + str(y_x[1]) + "]"
		if dx_dy is not None and dx_dy[0] != 0 and dx_dy[1] != 0:
			path = path + "%(" + str(dx_dy[0]) + "," + str(dx_dy[1]) + ")"
	return path


def _find_common_path(current_path, next_path):
	current_entry_list = get_entry_list(current_path)
	if len(current_entry_list) > 1:
		_, _, y_x, _ = get_entry(current_entry_list[-1])
		if (y_x is not None) and not is_int(y_x[0]):
			current_entry_list = get_entry_list(y_x[0])[:-1]
		else:
			current_entry_list = current_entry_list[:-1]
	next_entry_list = get_entry_list(next_path)
	if len(next_entry_list)>1:
		next_entry_list = next_entry_list[:-1]
	n = 0
	try:
		while current_entry_list[n] == next_entry_list[n]:
			n = n + 1
	except IndexError:
		common_path = path_separator.join(current_entry_list[0:n])
		return common_path
	common_path = path_separator.join(current_entry_list[0:n])
	return common_path


def _find_new_common_path_in_next_user_events(events, i):
	path_i = events[i].path
	i0 = i + 1
	new_common_path = ''
	while i0 < len(events):
		e = events[i0]
		if type(e) in (DragAndDropEvent, ClickEvent, FindEvent, MenuEvent):
			new_common_path = _find_common_path(path_i, e.path)
			break
		elif type(e) in (ElementEvent, mouse.MoveEvent):
			i0 = i0 + 1
		else:
			break
	if new_common_path == '':
		new_common_path = _find_common_path(path_i, path_i)
	return new_common_path


def _process_menu_select_events(events, i):
	i0 = i
	i_processed_events = []
	menu_path = []
	while i0 >= 0:
		if isinstance(events[i0], ClickEvent):
			entry_list = get_entry_list(events[i0].path)
			matching = [s for s in entry_list if "||MenuItem" in s]
			if matching:
				str_name, _, _, _ = get_entry(matching[0])
				menu_path.append(str_name)
				i_processed_events.append(i0)
				if [s for s in entry_list if "||MenuBar" in s]:
					break
			else:
				break
		i0 -= 1
	if menu_path:
		menu_path = path_separator.join(reversed(menu_path))
		i_menu_bar = i_processed_events.pop()
		menu_bar_path = get_entry_list(events[i_menu_bar].path)[0]
		events[i_menu_bar] = MenuEvent(path=menu_bar_path, menu_path=menu_path)
		for i_p_e in sorted(i_processed_events, reverse=True):
			del events[i_p_e]
			i -= 1
	return i


def _common_start(sa, sb):
	""" returns the longest common substring from the beginning of sa and sb """
	
	def _iter():
		for a, b in zip(sa, sb):
			if a == b:
				yield a
			else:
				return
	
	return ''.join(_iter())


def _get_typed_keys(keyboard_events):
	string = ''
	previous_event = None
	for event in keyboard_events:
		event_name = event.name.replace('windows gauche', 'left windows')
		event_name = event_name.replace('windows droite', 'right windows')
		if previous_event:
			common_event_name = _common_start(event.name, previous_event.name)
			if common_event_name:
				if previous_event.event_type == 'down' and event.event_type == 'up':
					if len(common_event_name) == 1:
						if previous_event and len(previous_event.name) == 1:
							string = string[:-len('""{? down}"')]
						else:
							string = string[:-len('{? down}"')]
						string = string + event_name + '"'
						previous_event = event
						continue
					else:
						string = string[:-len(' down}"')] + '}"'
						previous_event = event
						continue
		previous_event = event
		if event_name in keyboard.all_modifiers | {'maj', 'enter'}:
			string = string + '"' + "{VK_"
			if 'left' in event_name:
				string = string + "L"
			if 'right' in event_name or 'gr' in event_name:
				string = string + "R"
			if 'alt' in event_name:
				string = string + "MENU"
			elif 'ctrl' in event_name:
				string = string + "CONTROL"
			elif 'shift' in event_name or 'maj' in event_name:
				string = string + "SHIFT"
			elif 'windows' in event_name:
				string = string + "WIN"
			elif 'enter' in event_name:
				string = string[:-len("VK_")] + "ENTER"
			string = string + ' ' + event.event_type + "}" + '"'
		else:
			string = string + '"{' + event_name + ' ' + event.event_type + '}"'
	return string


def _get_typed_strings(keyboard_events, allow_backspace=True):
	"""
	Given a sequence of events, tries to deduce what strings were typed.
	Strings are separated when a non-textual key is pressed (such as tab or
	enter). Characters are converted to uppercase according to shift and
	capslock status. If `allow_backspace` is True, backspaces remove the last
	character typed. Control keys are converted into pywinauto.keyboard key codes
	"""
	backspace_name = 'backspace'
	
	shift_pressed = False
	capslock_pressed = False
	string = ''
	for event in keyboard_events:
		name = event.name
		
		# Space is the only key that we _parse_hotkey to the spelled out name
		# because of legibility. Now we have to undo that.
		if event.name == 'space':
			name = ' '
		
		if 'shift' in event.name:
			shift_pressed = event.event_type == 'down'
		elif event.name == 'caps lock' and event.event_type == 'down':
			capslock_pressed = not capslock_pressed
		elif allow_backspace and event.name == backspace_name and event.event_type == 'down':
			string = string[:-1]
		elif event.event_type == 'down':
			if len(name) == 1:
				if shift_pressed ^ capslock_pressed:
					name = name.upper()
				string = string + name
			else:
				if string:
					yield '"' + _escape_special_char(string) + '"'
				if 'windows' in event.name:
					yield '"' + '{LWIN}' + '"'
				elif 'enter' in event.name:
					yield '"' + '{ENTER}' + '"'
				string = ''


def _get_send_keys_strings(keyboard_events):
	is_typed_words = True
	alnum_count = 0
	for event in keyboard_events:
		if event.name in keyboard.all_modifiers:
			is_typed_words = False
			break
		if event.name.isalnum():
			alnum_count += 1
			if alnum_count > 1:
				break
	if alnum_count <= 1:
		is_typed_words = False
	if is_typed_words:
		return ''.join(format(code) for code in _get_typed_strings(keyboard_events))
	else:
		return _get_typed_keys(keyboard_events)


t0_progress_icon_timings = time.time()
progress_icon_timings = [0, 0, 0, 0, 0, 0, 0]


def _overlay_add_progress_icon(main_overlay, i, x, y):
	global t0_progress_icon_timings
	main_overlay.add(
		geometry=oaam.Shape.rectangle, x=x, y=y, width=52, height=52,
		color=(0, 0, 0), thickness=1, brush=oaam.Brush.solid, brush_color=(255, 255, 254))
	main_overlay.add(
		geometry=oaam.Shape.triangle,
		xyrgb_array=((x + 1, y + 1, 255, 255, 254), (x + 1, y + 52, 128, 128, 128), (x + 51, y + 52, 255, 255, 254)),
		thickness=0)
	dt = time.time() - t0_progress_icon_timings
	nb_dt = int(dt/0.01)
	if nb_dt > 255:
		nb_dt = int(255)
	progress_icon_timings[i % 6 -1] = nb_dt
	
	for b in range(i % 6):
		c = progress_icon_timings[b]
		main_overlay.add(
			geometry=oaam.Shape.rectangle, x=x + 6, y=y + 6 + b * 8, width=40, height=6,
			# color=(0, 255, 0), thickness=1, brush=oaam.Brush.solid, brush_color=(0, 200, 0))
			color=(c, int(255 - c/2), c ), thickness=1, brush=oaam.Brush.solid, brush_color=(c, int(255 - c), 0))
	t0_progress_icon_timings = time.time()


def _overlay_add_mode_icon(main_overlay, hicon, x, y):
	main_overlay.add(
		geometry=oaam.Shape.rectangle, x=x, y=y, width=52, height=52,
		color=(0, 0, 0), thickness=1, brush=oaam.Brush.solid, brush_color=(255, 255, 254))
	main_overlay.add(
		geometry=oaam.Shape.triangle,
		xyrgb_array=((x + 1, y + 1, 255, 255, 254), (x + 1, y + 52, 128, 128, 128), (x + 51, y + 52, 255, 255, 254)),
		thickness=0)
	main_overlay.add(
		geometry=oaam.Shape.image, hicon=hicon, x=int(x+2), y=int(y+2))


[docs]class Recorder(Thread): """ Recorder is a class thread used to record UI events in clipboard or in a file. .. code-block:: python :caption: Example of code using 'Recorder':: from pywinauto_recorder.recorder import Recorder from pywinauto_recorder.player import UIPath, click, move, playback recorder = Recorder() recorder.start_recording() with UIPath("Untitled - Notepad||Window"): doc = move("Text editor||Document") time.sleep(0.5) click(doc) utf8 = move("||Pane-> UTF-8||Text") time.sleep(0.5) click(utf8) recorded_python_script = recorder.stop_recording() recorder.quit() playback(filename=recorded_python_script) The above code clicks on the text field and then on 'utf8'. All these events are recorded in a file. Then the file is replayed. """ def __init__(self): from .element_observer import ElementInfoTooltip Thread.__init__(self) from win32api import GetSystemMetrics self._loop_t0 = None self.screen_width = GetSystemMetrics(0) self.screen_height = GetSystemMetrics(1) self.main_overlay = oaam.Overlay(transparency=0.4) self.desktop = pywinauto.Desktop(backend='uia', allow_magic_lookup=False) self.daemon = True self.event_list = [] self._copy_count = 0 self._mode = "Initializing" self._process_menu_click_mode = False self._smart_mode = False self._relative_coordinate_mode = False self.wrapper_old_info_tip = None self.common_path_info_tip = "" self.last_element_event = None self.started_recording_with_keyboard = False self.element_info_tooltip = ElementInfoTooltip() self.start() def __overlay_add_bold_rectangle(self, wrapper_rectangle, color=(0, 255, 0)): thickness = 5 r = wrapper_rectangle self.main_overlay.add( geometry=oaam.Shape.rectangle, x=r.left, y=r.top, width=r.width(), height=thickness, thickness=0, color=(0, 128, 0), brush=oaam.Brush.solid, brush_color=color) self.main_overlay.add( geometry=oaam.Shape.rectangle, x=r.left, y=r.bottom - thickness, width=r.width(), height=thickness, thickness=0, color=(0, 128, 0), brush=oaam.Brush.solid, brush_color=color) self.main_overlay.add( geometry=oaam.Shape.rectangle, x=r.left, y=r.top, width=thickness, height=r.height(), thickness=0, color=(0, 128, 0), brush=oaam.Brush.solid, brush_color=color) self.main_overlay.add( geometry=oaam.Shape.rectangle, x=r.right - thickness, y=r.top, width=thickness, height=r.height(), thickness=0, color=(0, 128, 0), brush=oaam.Brush.solid, brush_color=color) def __find_unique_element_array_1d(self, wrapper_rectangle, elements): nb_y, nb_x, candidates = get_sorted_region(elements) window_title = get_entry_list((get_wrapper_path(elements[0])))[0] for r_y in range(nb_y): for r_x in range(nb_x): try: r = candidates[r_y][r_x].rectangle() except IndexError: continue if r == wrapper_rectangle: xx, yy = r.left, r.mid_point()[1] previous_wrapper_path2 = None while xx > 0: # TODO: limiter la recherche Ă  la fenĂštre courante xx = xx - 9 wrapper2 = self.desktop.from_point(xx, yy) if wrapper2 is None: continue wrapper2_rectangle = wrapper2.rectangle() if wrapper2_rectangle.height() > wrapper_rectangle.height() * 2: continue wrapper_path2 = get_wrapper_path(wrapper2) if not wrapper_path2: continue if wrapper_path2 == previous_wrapper_path2: continue if get_entry_list(wrapper_path2)[0] != window_title: continue previous_wrapper_path2 = wrapper_path2 if find_elements(get_wrapper_path(wrapper2)): self.__overlay_add_bold_rectangle(wrapper2_rectangle, color=(0, 0, 255)) self.__overlay_add_bold_rectangle(wrapper_rectangle, color=(255, 200, 0)) return '#[' + wrapper_path2 + ',' + str(r_x) + ']' return None return None def __find_unique_element_array_2d(self, wrapper_rectangle, elements): nb_y, nb_x, candidates = get_sorted_region(elements) unique_array_2d = '' for r_y in range(nb_y): for r_x in range(nb_x): try: r = candidates[r_y][r_x].rectangle() except IndexError: continue if r == wrapper_rectangle: self.__overlay_add_bold_rectangle(r, color=(255, 200, 0)) unique_array_2d = '#[' + str(r_y) + ',' + str(r_x) + ']' else: self.__overlay_add_bold_rectangle(r, color=(255, 0, 0)) return unique_array_2d def __mouse_on(self, mouse_event): if self.mode == "Record": if isinstance(mouse_event, mouse.MoveEvent) and (len(self.event_list) > 0): if isinstance(self.event_list[-1], mouse.MoveEvent): self.event_list = self.event_list[:-1] self.event_list.append(mouse_event) def __start_stop_recording_by_key(self): if self.mode != "Record": self.started_recording_with_keyboard = True self.start_recording() else: self.stop_recording() def __start_stop_displaying_info_by_key(self): if self.mode == "Info": self.mode = "Stop" else: self.mode = "Info" def __display_found_elemenet_by_key(self): if self.last_element_event: self._copy_count = 2 x, y = win32api.GetCursorPos() l_e_e = self.last_element_event dx, dy = _compute_dx_dy(x, y, l_e_e.rectangle) str_dx, str_dy = "{:.2f}".format(round(dx * 100, 2)), "{:.2f}".format(round(dy * 100, 2)) i = l_e_e.path.find(path_separator) window_title = l_e_e.path[0:i] # element_path = l_e_e.path[i+len(path_separator):] p = _get_relative_path(window_title, l_e_e.path) code = 'with UIPath(u"' + _escape_special_char(window_title) + '"):\n' code += '\twrapper = find(u"' + _escape_special_char(p) if self.relative_coordinate_mode and eval(str_dx) != 0 and eval(str_dy) != 0: code += '%(' + str_dx + ',' + str_dy + ')' code += '")\n' code += '\twrapper.draw_outline()\n' pyperclip.copy(code) if self.event_list and self.mode == "Record": self.event_list.append(FindEvent(path=l_e_e.path, dx=dx, dy=dy, time=time.time())) def __key_on(self, e): key_to_scan_codes = keyboard.key_to_scan_codes if ( (e.name, e.event_type) == ('r', 'up') and set([key_to_scan_codes("alt")[0], key_to_scan_codes("ctrl")[0]]).issubset(keyboard._pressed_events)): self.__start_stop_recording_by_key() elif ( (e.name, e.event_type) == ('s', 'up') and set([key_to_scan_codes("alt")[0], key_to_scan_codes("ctrl")[0]]).issubset(keyboard._pressed_events)): self.smart_mode = not self.smart_mode elif ( (e.name, e.event_type) == ('F', 'up') and set([key_to_scan_codes("shift")[0], key_to_scan_codes("ctrl")[0]]).issubset(keyboard._pressed_events)): self.__display_found_elemenet_by_key() elif ( (e.name, e.event_type) == ('D', 'up') and set([key_to_scan_codes("shift")[0], key_to_scan_codes("ctrl")[0]]).issubset(keyboard._pressed_events)): self.__start_stop_displaying_info_by_key() elif self.mode == "Record": self.event_list.append(e) # Ne fonctionne pas dans tous les cas car un rectangle pere n'englobe pas forcement un rectangle fils # (par exemple un TreeItem) Mais peut ĂȘtre utilisĂ© en ultime recour ''' def my_from_point(self, x, y, wrapper=None): def get_children_of_children_whith_empty_rectangle(wrapper_children): children_of_children_whith_empty_rectangle = [] for child in wrapper_children: child_rectangle = child.rectangle() if child_rectangle.width() ==0 or child_rectangle.width()==0: children_of_children_whith_empty_rectangle += child.children() return children_of_children_whith_empty_rectangle def get_children_of_children_not_in_parent_rectangle(wrapper_children): children_of_children_whith_empty_rectangle = [] for child in wrapper_children: child_rectangle = child.rectangle() grandchildren = child.children() if grandchildren: x = grandchildren[0].rectangle().left y = grandchildren[0].rectangle().top if not ((child_rectangle.left < x < child_rectangle.right) and (child_rectangle.top < y < child_rectangle.bottom)): return grandchildren return [] if not wrapper: wrapper = self.desktop.top_from_point(x, y) wrapper_children = wrapper.children() for child in wrapper_children: child_rectangle = child.rectangle() if (child_rectangle.left < x < child_rectangle.right) and (child_rectangle.top < y < child_rectangle.bottom): return self.my_from_point(x, y, wrapper=child) for grandchild in get_children_of_children_whith_empty_rectangle(wrapper_children): child_rectangle = grandchild.rectangle() if (child_rectangle.left < x < child_rectangle.right) and (child_rectangle.top < y < child_rectangle.bottom): return self.my_from_point(x, y, wrapper=grandchild) for grandchild in get_children_of_children_not_in_parent_rectangle(wrapper_children): child_rectangle = grandchild.rectangle() if (child_rectangle.left < x < child_rectangle.right) and (child_rectangle.top < y < child_rectangle.bottom): return self.my_from_point(x, y, wrapper=grandchild) return wrapper '''
[docs] def run(self): """ The function is called in a loop, and it tries to find the unique element under the mouse cursor. """ import comtypes.client print("") print("COMPTYPES CACHE FOLDER:", comtypes.client._code_cache._find_gen_dir()) dir_path = os.path.dirname(os.path.realpath(__file__)) print("PYWINAUTO RECORDER FOLDER:", dir_path) keyboard.hook(self.__key_on) mouse.hook(self.__mouse_on) keyboard.start_recording() win32api.keybd_event(160, 0, win32con.KEYEVENTF_EXTENDEDKEY | win32con.KEYEVENTF_KEYUP, 0) ev_list = keyboard.stop_recording() if not ev_list and os.path.isfile(dir_path + r"\pywinauto_recorder.exe"): print("Couldn't set keyboard hooks. Trying once again...\n") time.sleep(0.5) os.system(dir_path + r"\pywinauto_recorder.exe --no_splash_screen") sys.exit(1) elements = [] i = 0 previous_wrapper_path = None unique_wrapper_path = None strategies = [Strategy.unique_path, Strategy.array_2D, Strategy.array_1D] i_strategy = 0 self.mode = "Info" #strategy_unique_path_again_done = False while self.mode != "Quit": i = i + 1 try: self._loop_t0 = time.time() self.main_overlay.clear_all() cursor_pos = win32api.GetCursorPos() wrapper = self.desktop.from_point(*cursor_pos) """ """ window = wrapper while window.parent() is not None and window.parent().parent() is not None: window = window.parent() set_native_window_handle(window.handle) """ """ #wrapper = self.my_from_point(*cursor_pos) if wrapper is None: time.sleep(0.01) continue wrapper_path = get_wrapper_path(wrapper) if not wrapper_path: time.sleep(0.01) continue if wrapper_path == previous_wrapper_path: if (unique_wrapper_path is None) or (strategies[i_strategy] == Strategy.array_2D): i_strategy = i_strategy + 1 if (not self.smart_mode) and (strategies[i_strategy] == Strategy.array_1D): i_strategy = 1 if i_strategy >= len(strategies): i_strategy = len(strategies) - 1 else: # strategy_unique_path_again_done = False i_strategy = 0 previous_wrapper_path = wrapper_path elements = find_elements(wrapper_path) #if wrapper_path == previous_wrapper_path and unique_wrapper_path: # strategy = Strategy.unique_path_again # else: # strategy = strategies[i_strategy] strategy = strategies[i_strategy] unique_wrapper_path = None # *** ----> this block of code must start a new while iteration if mouse cursor is outside wrapper rectangle # => add tests to leave if mouse cursor is outside wrapper rectangle wrapper_rectangle = wrapper.rectangle() #if strategy in [Strategy.unique_path, Strategy.unique_path_again]: if strategy is Strategy.unique_path: x_new, y_new = win32api.GetCursorPos() if not ((wrapper_rectangle.left < x_new < wrapper_rectangle.right) and ( wrapper_rectangle.top < y_new < wrapper_rectangle.bottom)): i_strategy = 0 continue if len(elements)==1: unique_wrapper_path = wrapper_path self.__overlay_add_bold_rectangle(wrapper_rectangle, color=(0, 255, 0)) else: for e in elements: self.__overlay_add_bold_rectangle(e.rectangle(), color=(255, 0, 0)) if strategy == Strategy.array_1D and elements: x_new, y_new = win32api.GetCursorPos() if not ((wrapper_rectangle.left < x_new < wrapper_rectangle.right) and ( wrapper_rectangle.top < y_new < wrapper_rectangle.bottom)): i_strategy = 0 continue unique_array_1d = self.__find_unique_element_array_1d(wrapper.rectangle(), elements) if unique_array_1d is not None: unique_wrapper_path = wrapper_path + unique_array_1d else: strategy = Strategy.array_2D if strategy == Strategy.array_2D and elements: x_new, y_new = win32api.GetCursorPos() if not ((wrapper_rectangle.left < x_new < wrapper_rectangle.right) and ( wrapper_rectangle.top < y_new < wrapper_rectangle.bottom)): i_strategy = 0 continue unique_array_2d = self.__find_unique_element_array_2d(wrapper.rectangle(), elements) if unique_array_2d is not None: unique_wrapper_path = wrapper_path + unique_array_2d # <----- *** if unique_wrapper_path is not None: #self.last_element_event = ElementEvent(strategy, wrapper.rectangle(), unique_wrapper_path) self.last_element_event = ElementEvent(strategy, wrapper_rectangle, unique_wrapper_path) if self.event_list and self.mode == "Record": self.event_list.append(self.last_element_event) nb_icons = 0 if self.mode == "Record": _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_record, 10, 10) nb_icons += 1 elif self.mode == "Stop": self.element_info_tooltip.hide() self.main_overlay.clear_all() _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_stop, 10, 10) self.main_overlay.refresh() while self.mode == "Stop": time.sleep(0.1) elif self.mode == "Play": self.element_info_tooltip.hide() self.main_overlay.clear_all() _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_play, 10, 10) self.main_overlay.refresh() while self.mode == "Play": time.sleep(1.0) if self.mode in ("Record", "Info"): _overlay_add_progress_icon(self.main_overlay, i, 10 + 60 * nb_icons, 10) nb_icons += 1 if self.mode == "Info": self.element_info_tooltip.show() if self.smart_mode: _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_light_on, 10 + 60 * nb_icons, 10) nb_icons += 1 if self._copy_count > 0: _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_clipboard, 10 + 60 * nb_icons, 10) nb_icons += 1 self._copy_count = self._copy_count - 1 self.main_overlay.refresh() loop_duration = time.time() - self._loop_t0 while loop_duration < 0.1: time.sleep(0.01) loop_duration = time.time() - self._loop_t0 time.sleep(0.01) # main_overlay.clear_all() doit attendre la fin de main_overlay.refresh() except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback))) self.common_path_info_tip = "" self.wrapper_old_info_tip = None if self.event_list: self.stop_recording() mouse.unhook_all() keyboard.unhook_all() self.main_overlay.quit() print("Run end")
@property def process_menu_click_mode(self): """ If True, the process menu events are recorded else they are ignored. :return: The state of the process menu click mode. """ return self._process_menu_click_mode @process_menu_click_mode.setter def process_menu_click_mode(self, value): """ It sets the state of the process menu click mode. :param value: If the value is True, the process menu events are recorded else they are ignored. """ self._process_menu_click_mode = value @property def relative_coordinate_mode(self): """ If True, the relative coordinates are recorded else they are ignored. :return: The state of the relative coordinates mode. """ return self._relative_coordinate_mode @relative_coordinate_mode.setter def relative_coordinate_mode(self, value): """ It sets the state of the relative coordinates mode. :param value: If the value is True, the relative coordinates are recorded else they are ignored. """ self._relative_coordinate_mode = value @property def smart_mode(self): """ If True, the smart mode is activated. :return: The state of the smart mode. """ return self._smart_mode @smart_mode.setter def smart_mode(self, value): """ It sets the state of the smart mode. :param value: If the value is True, the smart mode is activated else it is not activated. """ self._smart_mode = value @property def mode(self): """ It returns the mode of the recorder: "Record", "Play", "Info", "Stop", "Quit" :return: The mode of the recorder. """ return self._mode @mode.setter def mode(self, value): """ It sets the mode of the recorder: "Record", "Play", "Info", "Stop", "Quit" :param value: The mode of the recorder. """ self._mode = value
[docs] def start_recording(self): """ It adds a mouse move event to the event list, displays the record icon to the main overlay, clears and refreshes the main and info overlays, and then sets the mode to "Record". """ time.sleep(0.6) # wait the recorder to be fully ready x, y = win32api.GetCursorPos() self.event_list = [mouse.MoveEvent(x, y, time.time())] _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_record, 10, 10) self.element_info_tooltip.hide() self.main_overlay.clear_all() self.main_overlay.refresh() self.mode = "Record"
[docs] def stop_recording(self): """ It cleans the event list, displays the stop icon to the main overlay, clears and refreshes the main and info overlays, writes the Python script, and then sets the mode to "Stop". :return: The name of the file that was created. """ if self.mode == "Record" and len(self.event_list) > 2: events = list(self.event_list) self.event_list = [] self.mode = "Stop" time.sleep(0.6) # wait the recorder to be fully ready if self.started_recording_with_keyboard: _clean_events(events, remove_first_up=True) else: _clean_events(events) self.started_recording_with_keyboard = False _process_events(events, process_menu_click=self.process_menu_click_mode) _clean_events(events) return _write_in_file(events, relative_coordinate_mode=self.relative_coordinate_mode) self.main_overlay.clear_all() _overlay_add_mode_icon(self.main_overlay, IconSet.hicon_stop, 10, 10) self.main_overlay.refresh() self.mode = "Stop" return None
[docs] def get_last_element_event(self): """ It returns the last element of the event. :return: The last element event. """ return self.last_element_event
[docs] def playback(self, str_code='', filename=''): """ This function plays back a string of code or a Python file. :param str_code: The code to be played back :param filename: The name of the file coresponding to the code to be played back """ self.mode = "Play" self.main_overlay.refresh() playback(str_code, filename) self.mode = "Stop"
[docs] def quit(self): """ The function clears the main and info overlays, sets the mode to 'Quit', and then joins the thread. """ self.main_overlay.clear_all() self.main_overlay.refresh() del self.element_info_tooltip self.mode = 'Quit' self.join() print("Quit")
read_config_file()