diff --git a/stagehand/a11y/utils.py b/stagehand/a11y/utils.py index 6ea16f8b..d4768ed4 100644 --- a/stagehand/a11y/utils.py +++ b/stagehand/a11y/utils.py @@ -328,7 +328,7 @@ async def get_accessibility_tree( const pathIndex = hasSameTypeSiblings ? `[${index}]` : ""; parts.unshift(`${tagName}${pathIndex}`); } - + current = current.parentElement; } diff --git a/stagehand/utils.py b/stagehand/utils.py index 2383f46f..5d31eb1a 100644 --- a/stagehand/utils.py +++ b/stagehand/utils.py @@ -5,7 +5,7 @@ from pydantic import AnyUrl, BaseModel, Field, HttpUrl, create_model from pydantic.fields import FieldInfo -from stagehand.types.a11y import AccessibilityNode +from stagehand.types.a11y import AccessibilityNode, AXProperty, AXValue def snake_to_camel(snake_str: str) -> str: @@ -95,11 +95,86 @@ def convert_dict_keys_to_snake_case(data: Any) -> Any: return data +def _format_ax_value(value_type: str, value: AXValue) -> Union[str, None]: + """ + Formats the accessibility value, or returns None if the value is unsupported. + + NOTE: + Refer to "Accessible Rich Internet Applications (WAI-ARIA) 1.2" + for details. + https://www.w3.org/TR/wai-aria-1.2/#propcharacteristic_value + """ + if value_type == "tristate" and value in ["true", "mixed"]: + return str(value) + elif value_type == "booleanOrUndefined" and value in [True, "true"]: + return "true" + elif value_type == "number" and isinstance(value, (int, float)): + return str(value) + elif value_type == "string" and value: + return str(value) + return None + + +INCLUDED_NODE_PROPERTY_NAMES = { + "selected", + "checked", + "value", + "valuemin", + "valuemax", + "valuetext", + "valuenow", +} +""" +AX Property names included in the simplified tree. +""" + + +def _format_property(property: AXProperty) -> Union[str, None]: + name = property.get("name") + if name is None or (value_obj := property.get("value")) is None: + return None + value_type = value_obj["type"] + value = value_obj["value"] + value_formatted: Union[str, None] = None + + if (value_formatted := _format_ax_value(value_type, value)) is not None: + return f"{name}={value_formatted}" + return None + + +def _format_properties(node: AccessibilityNode) -> str: + """Formats the properties of a node into a simplified string representation.""" + included_properties: list[AXProperty] = [ + property + for property in (node.get("properties") or []) + if property["name"] in INCLUDED_NODE_PROPERTY_NAMES + ] + + formatted = ", ".join( + formatted + for property in included_properties + if (formatted := _format_property(property)) is not None + ) + + if formatted: + return f"({formatted})" + return "" + + def format_simplified_tree(node: AccessibilityNode, level: int = 0) -> str: """Formats a node and its children into a simplified string representation.""" indent = " " * level - name_part = f": {node.get('name')}" if node.get("name") else "" - result = f"{indent}[{node.get('nodeId')}] {node.get('role')}{name_part}\n" + name_part = f": {node_name.rstrip()}" if (node_name := node.get("name")) else "" + value_part = f" value={node.get('value')}" if node.get("value") else "" + properties_part = ( + f" {formatted_properties}" + if (formatted_properties := _format_properties(node)) + else "" + ) + result = ( + f"{indent}[{node.get('nodeId')}] {node.get('role')}{name_part}" + f"{value_part}{properties_part}\n" + ) children = node.get("children", []) if children: @@ -125,7 +200,7 @@ async def draw_observe_overlay(page, elements: list[dict]): (elements) => { // First remove any existing overlays document.querySelectorAll('.stagehand-observe-overlay').forEach(el => el.remove()); - + // Create container for overlays const container = document.createElement('div'); container.style.position = 'fixed'; @@ -137,7 +212,7 @@ async def draw_observe_overlay(page, elements: list[dict]): container.style.zIndex = '10000'; container.className = 'stagehand-observe-overlay'; document.body.appendChild(container); - + // Process each element elements.forEach((element, index) => { try { @@ -145,18 +220,18 @@ async def draw_observe_overlay(page, elements: list[dict]): let selector = element.selector; if (selector.startsWith('xpath=')) { selector = selector.substring(6); - + // Evaluate the XPath to get the element const result = document.evaluate( - selector, document, null, + selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null ); - + if (result.singleNodeValue) { // Get the element's position const el = result.singleNodeValue; const rect = el.getBoundingClientRect(); - + // Create the overlay const overlay = document.createElement('div'); overlay.style.position = 'absolute'; @@ -168,7 +243,7 @@ async def draw_observe_overlay(page, elements: list[dict]): overlay.style.backgroundColor = 'rgba(255, 0, 0, 0.1)'; overlay.style.boxSizing = 'border-box'; overlay.style.pointerEvents = 'none'; - + // Add element ID const label = document.createElement('div'); label.textContent = index + 1; @@ -180,7 +255,7 @@ async def draw_observe_overlay(page, elements: list[dict]): label.style.padding = '2px 5px'; label.style.borderRadius = '3px'; label.style.fontSize = '12px'; - + overlay.appendChild(label); container.appendChild(overlay); } @@ -189,7 +264,7 @@ async def draw_observe_overlay(page, elements: list[dict]): const el = document.querySelector(selector); if (el) { const rect = el.getBoundingClientRect(); - + // Create the overlay (same as above) const overlay = document.createElement('div'); overlay.style.position = 'absolute'; @@ -201,7 +276,7 @@ async def draw_observe_overlay(page, elements: list[dict]): overlay.style.backgroundColor = 'rgba(255, 0, 0, 0.1)'; overlay.style.boxSizing = 'border-box'; overlay.style.pointerEvents = 'none'; - + // Add element ID const label = document.createElement('div'); label.textContent = index + 1; @@ -213,7 +288,7 @@ async def draw_observe_overlay(page, elements: list[dict]): label.style.padding = '2px 5px'; label.style.borderRadius = '3px'; label.style.fontSize = '12px'; - + overlay.appendChild(label); container.appendChild(overlay); } @@ -222,7 +297,7 @@ async def draw_observe_overlay(page, elements: list[dict]): console.error(`Error drawing overlay for element ${index}:`, error); } }); - + // Auto-remove after 5 seconds setTimeout(() => { document.querySelectorAll('.stagehand-observe-overlay').forEach(el => el.remove()); diff --git a/tests/fixtures/ax_trees/input-radio.json b/tests/fixtures/ax_trees/input-radio.json new file mode 100644 index 00000000..5d5443ee --- /dev/null +++ b/tests/fixtures/ax_trees/input-radio.json @@ -0,0 +1,956 @@ +{ + "nodes": [ + { + "nodeId": "2", + "ignored": false, + "role": { + "type": "internalRole", + "value": "RootWebArea" + }, + "chromeRole": { + "type": "internalRole", + "value": 144 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "aria-label", + "superseded": true + }, + { + "type": "relatedElement", + "nativeSource": "title" + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "focused", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://example.com/input-radio.html" + } + } + ], + "childIds": [ + "3" + ], + "backendDOMNodeId": 2, + "frameId": "C94532EF60F347DD9D0B11AF1B1B2EB0" + }, + { + "nodeId": "3", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "2", + "childIds": [ + "7" + ], + "backendDOMNodeId": 3 + }, + { + "nodeId": "7", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "3", + "childIds": [ + "8" + ], + "backendDOMNodeId": 7 + }, + { + "nodeId": "8", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 211 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "7", + "childIds": [ + "9", + "10" + ], + "backendDOMNodeId": 8 + }, + { + "nodeId": "9", + "ignored": false, + "role": { + "type": "role", + "value": "heading" + }, + "chromeRole": { + "type": "internalRole", + "value": 96 + }, + "name": { + "type": "computedString", + "value": "Radio Menus", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Radio Menus" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "8", + "childIds": [ + "21" + ], + "backendDOMNodeId": 9 + }, + { + "nodeId": "10", + "ignored": false, + "role": { + "type": "role", + "value": "group" + }, + "chromeRole": { + "type": "internalRole", + "value": 93 + }, + "name": { + "type": "computedString", + "value": "Select a maintenance drone:", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "relatedElement", + "value": { + "type": "computedString", + "value": "Select a maintenance drone:" + }, + "nativeSource": "legend", + "nativeSourceValue": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 11, + "text": "Select a maintenance drone:" + } + ] + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "invalid", + "value": { + "type": "token", + "value": "false" + } + }, + { + "name": "labelledby", + "value": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 11, + "text": "Select a maintenance drone:" + } + ] + } + } + ], + "parentId": "8", + "childIds": [ + "11", + "12", + "15", + "18" + ], + "backendDOMNodeId": 10 + }, + { + "nodeId": "21", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Radio Menus", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Radio Menus" + } + } + ] + }, + "properties": [], + "parentId": "9", + "childIds": [ + "-1000000002" + ], + "backendDOMNodeId": 21 + }, + { + "nodeId": "11", + "ignored": false, + "role": { + "type": "internalRole", + "value": "Legend" + }, + "chromeRole": { + "type": "internalRole", + "value": 108 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "10", + "childIds": [ + "22" + ], + "backendDOMNodeId": 11 + }, + { + "nodeId": "12", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 88 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + } + ] + }, + "properties": [], + "parentId": "10", + "childIds": [ + "13", + "14" + ], + "backendDOMNodeId": 12 + }, + { + "nodeId": "15", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 88 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + } + ] + }, + "properties": [], + "parentId": "10", + "childIds": [ + "16", + "17" + ], + "backendDOMNodeId": 15 + }, + { + "nodeId": "18", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 88 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + } + ] + }, + "properties": [], + "parentId": "10", + "childIds": [ + "19", + "20" + ], + "backendDOMNodeId": 18 + }, + { + "nodeId": "-1000000002", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Radio Menus" + }, + "properties": [], + "parentId": "21", + "childIds": [] + }, + { + "nodeId": "22", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Select a maintenance drone:", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Select a maintenance drone:" + } + } + ] + }, + "properties": [], + "parentId": "11", + "childIds": [ + "-1000000003" + ], + "backendDOMNodeId": 22 + }, + { + "nodeId": "13", + "ignored": false, + "role": { + "type": "role", + "value": "radio" + }, + "chromeRole": { + "type": "internalRole", + "value": 141 + }, + "name": { + "type": "computedString", + "value": "Huey", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "relatedElement", + "value": { + "type": "computedString", + "value": "Huey" + }, + "nativeSource": "labelfor", + "nativeSourceValue": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 14, + "text": "Huey" + } + ] + } + }, + { + "type": "contents", + "superseded": true + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "invalid", + "value": { + "type": "token", + "value": "false" + } + }, + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "checked", + "value": { + "type": "tristate", + "value": "true" + } + }, + { + "name": "labelledby", + "value": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 14, + "text": "Huey" + } + ] + } + } + ], + "parentId": "12", + "childIds": [], + "backendDOMNodeId": 13 + }, + { + "nodeId": "14", + "ignored": true, + "ignoredReasons": [ + { + "name": "labelFor", + "value": { + "type": "idref", + "relatedNodes": [ + { + "backendDOMNodeId": 13, + "idref": "huey" + } + ] + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "12", + "childIds": [ + "23" + ], + "backendDOMNodeId": 14 + }, + { + "nodeId": "23", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "14", + "childIds": [], + "backendDOMNodeId": 23 + }, + { + "nodeId": "16", + "ignored": false, + "role": { + "type": "role", + "value": "radio" + }, + "chromeRole": { + "type": "internalRole", + "value": 141 + }, + "name": { + "type": "computedString", + "value": "Dewey", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "relatedElement", + "value": { + "type": "computedString", + "value": "Dewey" + }, + "nativeSource": "labelfor", + "nativeSourceValue": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 17, + "text": "Dewey" + } + ] + } + }, + { + "type": "contents", + "superseded": true + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "invalid", + "value": { + "type": "token", + "value": "false" + } + }, + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "checked", + "value": { + "type": "tristate", + "value": "false" + } + }, + { + "name": "labelledby", + "value": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 17, + "text": "Dewey" + } + ] + } + } + ], + "parentId": "15", + "childIds": [], + "backendDOMNodeId": 16 + }, + { + "nodeId": "17", + "ignored": true, + "ignoredReasons": [ + { + "name": "labelFor", + "value": { + "type": "idref", + "relatedNodes": [ + { + "backendDOMNodeId": 16, + "idref": "dewey" + } + ] + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "15", + "childIds": [ + "24" + ], + "backendDOMNodeId": 17 + }, + { + "nodeId": "24", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "17", + "childIds": [], + "backendDOMNodeId": 24 + }, + { + "nodeId": "19", + "ignored": false, + "role": { + "type": "role", + "value": "radio" + }, + "chromeRole": { + "type": "internalRole", + "value": 141 + }, + "name": { + "type": "computedString", + "value": "Louie", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "relatedElement", + "value": { + "type": "computedString", + "value": "Louie" + }, + "nativeSource": "labelfor", + "nativeSourceValue": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 20, + "text": "Louie" + } + ] + } + }, + { + "type": "contents", + "superseded": true + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "invalid", + "value": { + "type": "token", + "value": "false" + } + }, + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "checked", + "value": { + "type": "tristate", + "value": "false" + } + }, + { + "name": "labelledby", + "value": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 20, + "text": "Louie" + } + ] + } + } + ], + "parentId": "18", + "childIds": [], + "backendDOMNodeId": 19 + }, + { + "nodeId": "20", + "ignored": true, + "ignoredReasons": [ + { + "name": "labelFor", + "value": { + "type": "idref", + "relatedNodes": [ + { + "backendDOMNodeId": 19, + "idref": "louie" + } + ] + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "18", + "childIds": [ + "25" + ], + "backendDOMNodeId": 20 + }, + { + "nodeId": "25", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "20", + "childIds": [], + "backendDOMNodeId": 25 + }, + { + "nodeId": "-1000000003", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Select a maintenance drone:" + }, + "properties": [], + "parentId": "22", + "childIds": [] + } + ] +} + diff --git a/tests/fixtures/ax_trees/input-range.json b/tests/fixtures/ax_trees/input-range.json new file mode 100644 index 00000000..d0dea169 --- /dev/null +++ b/tests/fixtures/ax_trees/input-range.json @@ -0,0 +1,495 @@ +{ + "nodes": [ + { + "nodeId": "2", + "ignored": false, + "role": { + "type": "internalRole", + "value": "RootWebArea" + }, + "chromeRole": { + "type": "internalRole", + "value": 144 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "aria-label", + "superseded": true + }, + { + "type": "relatedElement", + "nativeSource": "title" + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "focused", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://example.com/input-range.html" + } + } + ], + "childIds": [ + "3" + ], + "backendDOMNodeId": 2, + "frameId": "4C8AFBFB04749B93F2BA71C9854C9B01" + }, + { + "nodeId": "3", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "2", + "childIds": [ + "7" + ], + "backendDOMNodeId": 3 + }, + { + "nodeId": "7", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "3", + "childIds": [ + "8" + ], + "backendDOMNodeId": 7 + }, + { + "nodeId": "8", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 211 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "7", + "childIds": [ + "9", + "10" + ], + "backendDOMNodeId": 8 + }, + { + "nodeId": "9", + "ignored": false, + "role": { + "type": "role", + "value": "heading" + }, + "chromeRole": { + "type": "internalRole", + "value": 96 + }, + "name": { + "type": "computedString", + "value": "Range Input", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Range Input" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "8", + "childIds": [ + "16" + ], + "backendDOMNodeId": 9 + }, + { + "nodeId": "10", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 88 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + } + ] + }, + "properties": [], + "parentId": "8", + "childIds": [ + "11", + "15" + ], + "backendDOMNodeId": 10 + }, + { + "nodeId": "16", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Range Input", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Range Input" + } + } + ] + }, + "properties": [], + "parentId": "9", + "childIds": [ + "-1000000002" + ], + "backendDOMNodeId": 16 + }, + { + "nodeId": "11", + "ignored": false, + "role": { + "type": "role", + "value": "slider" + }, + "chromeRole": { + "type": "internalRole", + "value": 155 + }, + "name": { + "type": "computedString", + "value": "Volume", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "relatedElement", + "value": { + "type": "computedString", + "value": "Volume" + }, + "nativeSource": "labelfor", + "nativeSourceValue": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 15, + "text": "Volume" + } + ] + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "value": { + "type": "number", + "value": 5 + }, + "properties": [ + { + "name": "invalid", + "value": { + "type": "token", + "value": "false" + } + }, + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "settable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "orientation", + "value": { + "type": "token", + "value": "horizontal" + } + }, + { + "name": "valuemin", + "value": { + "type": "number", + "value": 0 + } + }, + { + "name": "valuemax", + "value": { + "type": "number", + "value": 11 + } + }, + { + "name": "valuetext", + "value": { + "type": "string", + "value": "" + } + }, + { + "name": "labelledby", + "value": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 15, + "text": "Volume" + } + ] + } + } + ], + "parentId": "10", + "childIds": [], + "backendDOMNodeId": 11 + }, + { + "nodeId": "15", + "ignored": false, + "role": { + "type": "internalRole", + "value": "LabelText" + }, + "chromeRole": { + "type": "internalRole", + "value": 104 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "10", + "childIds": [ + "17" + ], + "backendDOMNodeId": 15 + }, + { + "nodeId": "-1000000002", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Range Input" + }, + "properties": [], + "parentId": "16", + "childIds": [] + }, + { + "nodeId": "17", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Volume", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Volume" + } + } + ] + }, + "properties": [], + "parentId": "15", + "childIds": [ + "-1000000003" + ], + "backendDOMNodeId": 17 + }, + { + "nodeId": "-1000000003", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Volume" + }, + "properties": [], + "parentId": "17", + "childIds": [] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ax_trees/list_links.json b/tests/fixtures/ax_trees/list_links.json new file mode 100644 index 00000000..8e134cdb --- /dev/null +++ b/tests/fixtures/ax_trees/list_links.json @@ -0,0 +1,1129 @@ +{ + "nodes": [ + { + "nodeId": "2", + "ignored": false, + "role": { + "type": "internalRole", + "value": "RootWebArea" + }, + "chromeRole": { + "type": "internalRole", + "value": 144 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "aria-label", + "superseded": true + }, + { + "type": "relatedElement", + "nativeSource": "title" + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "focused", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://example.com/list_links.html" + } + } + ], + "childIds": [ + "3" + ], + "backendDOMNodeId": 2, + "frameId": "96BBC88CE9967B36BBD41CFCBFD0FF1F" + }, + { + "nodeId": "3", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "2", + "childIds": [ + "7" + ], + "backendDOMNodeId": 3 + }, + { + "nodeId": "7", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "3", + "childIds": [ + "8" + ], + "backendDOMNodeId": 7 + }, + { + "nodeId": "8", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 211 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "7", + "childIds": [ + "9", + "10" + ], + "backendDOMNodeId": 8 + }, + { + "nodeId": "9", + "ignored": false, + "role": { + "type": "role", + "value": "heading" + }, + "chromeRole": { + "type": "internalRole", + "value": 96 + }, + "name": { + "type": "computedString", + "value": "List Links", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "List Links" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "8", + "childIds": [ + "19" + ], + "backendDOMNodeId": 9 + }, + { + "nodeId": "10", + "ignored": false, + "role": { + "type": "role", + "value": "list" + }, + "chromeRole": { + "type": "internalRole", + "value": 111 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "8", + "childIds": [ + "11", + "13", + "15", + "17" + ], + "backendDOMNodeId": 10 + }, + { + "nodeId": "19", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "List Links", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "List Links" + } + } + ] + }, + "properties": [], + "parentId": "9", + "childIds": [ + "-1000000002" + ], + "backendDOMNodeId": 19 + }, + { + "nodeId": "11", + "ignored": false, + "role": { + "type": "role", + "value": "listitem" + }, + "chromeRole": { + "type": "internalRole", + "value": 115 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "10", + "childIds": [ + "20", + "12" + ], + "backendDOMNodeId": 11 + }, + { + "nodeId": "13", + "ignored": false, + "role": { + "type": "role", + "value": "listitem" + }, + "chromeRole": { + "type": "internalRole", + "value": 115 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "10", + "childIds": [ + "22", + "14" + ], + "backendDOMNodeId": 13 + }, + { + "nodeId": "15", + "ignored": false, + "role": { + "type": "role", + "value": "listitem" + }, + "chromeRole": { + "type": "internalRole", + "value": 115 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "10", + "childIds": [ + "24", + "16" + ], + "backendDOMNodeId": 15 + }, + { + "nodeId": "17", + "ignored": false, + "role": { + "type": "role", + "value": "listitem" + }, + "chromeRole": { + "type": "internalRole", + "value": 115 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "10", + "childIds": [ + "26", + "18" + ], + "backendDOMNodeId": 17 + }, + { + "nodeId": "-1000000002", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "List Links" + }, + "properties": [], + "parentId": "19", + "childIds": [] + }, + { + "nodeId": "20", + "ignored": false, + "role": { + "type": "internalRole", + "value": "ListMarker" + }, + "chromeRole": { + "type": "internalRole", + "value": 116 + }, + "name": { + "type": "computedString", + "value": "\u2022 ", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "\u2022 " + } + } + ] + }, + "properties": [], + "parentId": "11", + "childIds": [ + "-1000000003" + ], + "backendDOMNodeId": 20 + }, + { + "nodeId": "12", + "ignored": false, + "role": { + "type": "role", + "value": "link" + }, + "chromeRole": { + "type": "internalRole", + "value": 110 + }, + "name": { + "type": "computedString", + "value": "Google", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Google" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://www.google.com/" + } + } + ], + "parentId": "11", + "childIds": [ + "21" + ], + "backendDOMNodeId": 12 + }, + { + "nodeId": "22", + "ignored": false, + "role": { + "type": "internalRole", + "value": "ListMarker" + }, + "chromeRole": { + "type": "internalRole", + "value": 116 + }, + "name": { + "type": "computedString", + "value": "\u2022 ", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "\u2022 " + } + } + ] + }, + "properties": [], + "parentId": "13", + "childIds": [ + "-1000000005" + ], + "backendDOMNodeId": 22 + }, + { + "nodeId": "14", + "ignored": false, + "role": { + "type": "role", + "value": "link" + }, + "chromeRole": { + "type": "internalRole", + "value": 110 + }, + "name": { + "type": "computedString", + "value": "DuckDuckGo", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "DuckDuckGo" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://duckduckgo.com/" + } + } + ], + "parentId": "13", + "childIds": [ + "23" + ], + "backendDOMNodeId": 14 + }, + { + "nodeId": "24", + "ignored": false, + "role": { + "type": "internalRole", + "value": "ListMarker" + }, + "chromeRole": { + "type": "internalRole", + "value": 116 + }, + "name": { + "type": "computedString", + "value": "\u2022 ", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "\u2022 " + } + } + ] + }, + "properties": [], + "parentId": "15", + "childIds": [ + "-1000000007" + ], + "backendDOMNodeId": 24 + }, + { + "nodeId": "16", + "ignored": false, + "role": { + "type": "role", + "value": "link" + }, + "chromeRole": { + "type": "internalRole", + "value": 110 + }, + "name": { + "type": "computedString", + "value": "Bing", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Bing" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://www.bing.com/" + } + } + ], + "parentId": "15", + "childIds": [ + "25" + ], + "backendDOMNodeId": 16 + }, + { + "nodeId": "26", + "ignored": false, + "role": { + "type": "internalRole", + "value": "ListMarker" + }, + "chromeRole": { + "type": "internalRole", + "value": 116 + }, + "name": { + "type": "computedString", + "value": "\u2022 ", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "\u2022 " + } + } + ] + }, + "properties": [], + "parentId": "17", + "childIds": [ + "-1000000009" + ], + "backendDOMNodeId": 26 + }, + { + "nodeId": "18", + "ignored": false, + "role": { + "type": "role", + "value": "link" + }, + "chromeRole": { + "type": "internalRole", + "value": 110 + }, + "name": { + "type": "computedString", + "value": "Brave", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Brave" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://search.brave.com/" + } + } + ], + "parentId": "17", + "childIds": [ + "27" + ], + "backendDOMNodeId": 18 + }, + { + "nodeId": "-1000000003", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "20", + "childIds": [] + }, + { + "nodeId": "21", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Google", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Google" + } + } + ] + }, + "properties": [], + "parentId": "12", + "childIds": [ + "-1000000004" + ], + "backendDOMNodeId": 21 + }, + { + "nodeId": "-1000000005", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "22", + "childIds": [] + }, + { + "nodeId": "23", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "DuckDuckGo", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "DuckDuckGo" + } + } + ] + }, + "properties": [], + "parentId": "14", + "childIds": [ + "-1000000006" + ], + "backendDOMNodeId": 23 + }, + { + "nodeId": "-1000000007", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "24", + "childIds": [] + }, + { + "nodeId": "25", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Bing", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Bing" + } + } + ] + }, + "properties": [], + "parentId": "16", + "childIds": [ + "-1000000008" + ], + "backendDOMNodeId": 25 + }, + { + "nodeId": "-1000000009", + "ignored": true, + "ignoredReasons": [ + { + "name": "presentationalRole", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "26", + "childIds": [] + }, + { + "nodeId": "27", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Brave", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Brave" + } + } + ] + }, + "properties": [], + "parentId": "18", + "childIds": [ + "-1000000010" + ], + "backendDOMNodeId": 27 + }, + { + "nodeId": "-1000000004", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Google" + }, + "properties": [], + "parentId": "21", + "childIds": [] + }, + { + "nodeId": "-1000000006", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "DuckDuckGo" + }, + "properties": [], + "parentId": "23", + "childIds": [] + }, + { + "nodeId": "-1000000008", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Bing" + }, + "properties": [], + "parentId": "25", + "childIds": [] + }, + { + "nodeId": "-1000000010", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Brave" + }, + "properties": [], + "parentId": "27", + "childIds": [] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ax_trees/select.json b/tests/fixtures/ax_trees/select.json new file mode 100644 index 00000000..470ec0bd --- /dev/null +++ b/tests/fixtures/ax_trees/select.json @@ -0,0 +1,653 @@ +{ + "nodes": [ + { + "nodeId": "2", + "ignored": false, + "role": { + "type": "internalRole", + "value": "RootWebArea" + }, + "chromeRole": { + "type": "internalRole", + "value": 144 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "aria-label", + "superseded": true + }, + { + "type": "relatedElement", + "nativeSource": "title" + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "focused", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "url", + "value": { + "type": "string", + "value": "https://example.com/select.html" + } + } + ], + "childIds": [ + "4" + ], + "backendDOMNodeId": 2, + "frameId": "A1A3C066FAD63FDFE9CA5E334C6A5E3A" + }, + { + "nodeId": "4", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "2", + "childIds": [ + "8" + ], + "backendDOMNodeId": 4 + }, + { + "nodeId": "8", + "ignored": true, + "ignoredReasons": [ + { + "name": "uninteresting", + "value": { + "type": "boolean", + "value": true + } + } + ], + "role": { + "type": "role", + "value": "none" + }, + "chromeRole": { + "type": "internalRole", + "value": 0 + }, + "parentId": "4", + "childIds": [ + "9" + ], + "backendDOMNodeId": 8 + }, + { + "nodeId": "9", + "ignored": false, + "role": { + "type": "role", + "value": "generic" + }, + "chromeRole": { + "type": "internalRole", + "value": 211 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "8", + "childIds": [ + "10", + "11", + "12" + ], + "backendDOMNodeId": 9 + }, + { + "nodeId": "10", + "ignored": false, + "role": { + "type": "role", + "value": "heading" + }, + "chromeRole": { + "type": "internalRole", + "value": 96 + }, + "name": { + "type": "computedString", + "value": "Select Menus", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Select Menus" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "level", + "value": { + "type": "integer", + "value": 1 + } + } + ], + "parentId": "9", + "childIds": [ + "28" + ], + "backendDOMNodeId": 10 + }, + { + "nodeId": "11", + "ignored": false, + "role": { + "type": "internalRole", + "value": "LabelText" + }, + "chromeRole": { + "type": "internalRole", + "value": 104 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "9", + "childIds": [ + "29" + ], + "backendDOMNodeId": 11 + }, + { + "nodeId": "12", + "ignored": false, + "role": { + "type": "role", + "value": "combobox" + }, + "chromeRole": { + "type": "internalRole", + "value": 209 + }, + "name": { + "type": "computedString", + "value": "Choose a pet:", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "relatedElement", + "value": { + "type": "computedString", + "value": "Choose a pet:" + }, + "nativeSource": "labelfor", + "nativeSourceValue": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 11, + "text": "Choose a pet:" + } + ] + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "value": { + "type": "string", + "value": "Hamster" + }, + "properties": [ + { + "name": "invalid", + "value": { + "type": "token", + "value": "false" + } + }, + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "hasPopup", + "value": { + "type": "token", + "value": "menu" + } + }, + { + "name": "expanded", + "value": { + "type": "booleanOrUndefined", + "value": false + } + }, + { + "name": "labelledby", + "value": { + "type": "nodeList", + "relatedNodes": [ + { + "backendDOMNodeId": 11, + "text": "Choose a pet:" + } + ] + } + } + ], + "parentId": "9", + "childIds": [ + "15" + ], + "backendDOMNodeId": 12 + }, + { + "nodeId": "28", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Select Menus", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Select Menus" + } + } + ] + }, + "properties": [], + "parentId": "10", + "childIds": [ + "-1000000002" + ], + "backendDOMNodeId": 28 + }, + { + "nodeId": "29", + "ignored": false, + "role": { + "type": "internalRole", + "value": "StaticText" + }, + "chromeRole": { + "type": "internalRole", + "value": 158 + }, + "name": { + "type": "computedString", + "value": "Choose a pet:", + "sources": [ + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Choose a pet:" + } + } + ] + }, + "properties": [], + "parentId": "11", + "childIds": [ + "-1000000003" + ], + "backendDOMNodeId": 29 + }, + { + "nodeId": "15", + "ignored": false, + "role": { + "type": "internalRole", + "value": "MenuListPopup" + }, + "chromeRole": { + "type": "internalRole", + "value": 128 + }, + "name": { + "type": "computedString", + "value": "", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "attribute", + "attribute": "title" + } + ] + }, + "properties": [], + "parentId": "12", + "childIds": [ + "19", + "22", + "25" + ], + "backendDOMNodeId": 15 + }, + { + "nodeId": "-1000000002", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Select Menus" + }, + "properties": [], + "parentId": "28", + "childIds": [] + }, + { + "nodeId": "-1000000003", + "ignored": false, + "role": { + "type": "internalRole", + "value": "InlineTextBox" + }, + "chromeRole": { + "type": "internalRole", + "value": 101 + }, + "name": { + "type": "computedString", + "value": "Choose a pet:" + }, + "properties": [], + "parentId": "29", + "childIds": [] + }, + { + "nodeId": "19", + "ignored": false, + "role": { + "type": "role", + "value": "option" + }, + "chromeRole": { + "type": "internalRole", + "value": 127 + }, + "name": { + "type": "computedString", + "value": "Dog", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Dog" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "selected", + "value": { + "type": "booleanOrUndefined", + "value": false + } + } + ], + "parentId": "15", + "childIds": [], + "backendDOMNodeId": 19 + }, + { + "nodeId": "22", + "ignored": false, + "role": { + "type": "role", + "value": "option" + }, + "chromeRole": { + "type": "internalRole", + "value": 127 + }, + "name": { + "type": "computedString", + "value": "Cat", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Cat" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "selected", + "value": { + "type": "booleanOrUndefined", + "value": false + } + } + ], + "parentId": "15", + "childIds": [], + "backendDOMNodeId": 22 + }, + { + "nodeId": "25", + "ignored": false, + "role": { + "type": "role", + "value": "option" + }, + "chromeRole": { + "type": "internalRole", + "value": 127 + }, + "name": { + "type": "computedString", + "value": "Hamster", + "sources": [ + { + "type": "relatedElement", + "attribute": "aria-labelledby" + }, + { + "type": "attribute", + "attribute": "aria-label" + }, + { + "type": "contents", + "value": { + "type": "computedString", + "value": "Hamster" + } + }, + { + "type": "attribute", + "attribute": "title", + "superseded": true + } + ] + }, + "properties": [ + { + "name": "focusable", + "value": { + "type": "booleanOrUndefined", + "value": true + } + }, + { + "name": "selected", + "value": { + "type": "booleanOrUndefined", + "value": true + } + } + ], + "parentId": "15", + "childIds": [], + "backendDOMNodeId": 25 + } + ] +} diff --git a/tests/fixtures/html_pages/input-radio.html b/tests/fixtures/html_pages/input-radio.html new file mode 100644 index 00000000..185a6968 --- /dev/null +++ b/tests/fixtures/html_pages/input-radio.html @@ -0,0 +1,30 @@ + + + + + + + +
+

Radio Menus

+
+ Select a maintenance drone: +
+ + +
+ +
+ + +
+ +
+ + +
+
+
+ + + diff --git a/tests/fixtures/html_pages/input-range.html b/tests/fixtures/html_pages/input-range.html new file mode 100644 index 00000000..d5613d0f --- /dev/null +++ b/tests/fixtures/html_pages/input-range.html @@ -0,0 +1,16 @@ + + + + + + + +
+

Range Input

+
+ + +
+
+ + diff --git a/tests/fixtures/html_pages/list_links.html b/tests/fixtures/html_pages/list_links.html new file mode 100644 index 00000000..aeadd9af --- /dev/null +++ b/tests/fixtures/html_pages/list_links.html @@ -0,0 +1,34 @@ + + + + + + + +
+

List Links

+ +
+ + diff --git a/tests/fixtures/html_pages/select.html b/tests/fixtures/html_pages/select.html new file mode 100644 index 00000000..f1f59ab5 --- /dev/null +++ b/tests/fixtures/html_pages/select.html @@ -0,0 +1,18 @@ + + + + + + + +
+

Select Menus

+ + +
+ + diff --git a/tests/unit/a11y/test_utils.py b/tests/unit/a11y/test_utils.py new file mode 100644 index 00000000..b1e84ca0 --- /dev/null +++ b/tests/unit/a11y/test_utils.py @@ -0,0 +1,170 @@ +from typing import Any +from unittest.mock import MagicMock, patch +import json + +import pytest + +from stagehand import StagehandPage +from stagehand.a11y import get_accessibility_tree +from stagehand.logging import StagehandLogger + +pytestmark = [pytest.mark.asyncio] + + +@pytest.fixture +def mock_stagehand_logger(): + with patch('stagehand.a11y.utils.StagehandLogger'): + mock_logger = MagicMock() + yield mock_logger + + +@pytest.fixture +def load_ax_tree(pytestconfig): + def loader(name: str) -> dict[str, Any]: + fixture = pytestconfig.rootpath / "tests/fixtures/ax_trees" / name + return json.loads(fixture.read_text()) + return loader + + + +@pytest.fixture +def mock_send_cdp(mock_stagehand_page): + def mock_send_cdp_factory(*, ax_tree, backend_nodes=None, tag_names=None): + backend_nodes = backend_nodes or {} + tag_names = tag_names or {} + + async def mocked_send_cdp(method, params=None): + params = params or {} + if method == "Accessibility.getFullAXTree": + return ax_tree + elif method == "DOM.resolveNode": + # Create a mapping of element IDs to appropriate object IDs + backend_node_id = params.get("backendNodeId", 1) + return backend_nodes.get(backend_node_id, {}) + elif method == "Runtime.callFunctionOn": + object_id = params.get("objectId") + functionDeclaration = params.get("functionDeclaration") + if functionDeclaration != 'function() { return this.tagName ? this.tagName.toLowerCase() : ""; }': + raise NotImplementedError + return tag_names.get(object_id, {}) + else: + return {} + + mock_stagehand_page.send_cdp.side_effect = mocked_send_cdp + return mocked_send_cdp + + return mock_send_cdp_factory + + +class TestGetAccessibilityTree: + async def test_empty_tree_result(self, mock_stagehand_page: StagehandPage, mock_stagehand_logger: StagehandLogger, mock_send_cdp): + mock_send_cdp(ax_tree={"nodes": []}) + actual = await get_accessibility_tree(mock_stagehand_page, mock_stagehand_logger) + assert actual["simplified"] == "" + assert actual["tree"] == [] + assert actual["iframes"] == [] + assert actual["idToUrl"] == {} + + async def test_select_tag_included_in_simplified(self, mock_stagehand_page: StagehandPage, mock_stagehand_logger: StagehandLogger, mock_send_cdp, load_ax_tree): + mock_send_cdp( + ax_tree=load_ax_tree("select.json"), + backend_nodes={12: {"object": {"objectId": "1234"}}}, + tag_names={"1234": {"result": {"value": "select"}}}, + ) + actual = await get_accessibility_tree(mock_stagehand_page, mock_stagehand_logger) + + assert ( +"""[2] RootWebArea + [9] generic + [10] heading: Select Menus + [11] LabelText + [29] StaticText: Choose a pet: + [12] select: Choose a pet: value=Hamster + [15] MenuListPopup + [19] option: Dog + [22] option: Cat + [25] option: Hamster (selected=true) +""" + ) == actual["simplified"] + assert [] == actual["iframes"] + assert {"2": "https://example.com/select.html"} == actual["idToUrl"] + + async def test_input_type_radio(self, mock_stagehand_page: StagehandPage, mock_stagehand_logger: StagehandLogger, mock_send_cdp, load_ax_tree): + mock_send_cdp( + ax_tree=load_ax_tree("input-radio.json"), + backend_nodes={12: {"object": {"objectId": "1234"}}}, + tag_names={"1234": {"result": {"value": "select"}}}, + ) + actual = await get_accessibility_tree(mock_stagehand_page, mock_stagehand_logger) + + assert ( +"""[2] RootWebArea + [8] generic + [9] heading: Radio Menus + [10] group: Select a maintenance drone: + [11] Legend + [22] StaticText: Select a maintenance drone: + [13] radio: Huey (checked=true) + [16] radio: Dewey + [19] radio: Louie +""" + ) == actual["simplified"] + assert [] == actual["iframes"] + assert {"2": "https://example.com/input-radio.html"} == actual["idToUrl"] + + async def test_input_type_range(self, mock_stagehand_page: StagehandPage, mock_stagehand_logger: StagehandLogger, mock_send_cdp, load_ax_tree): + mock_send_cdp( + ax_tree=load_ax_tree("input-range.json"), + backend_nodes={12: {"object": {"objectId": "1234"}}}, + tag_names={"1234": {"result": {"value": "select"}}}, + ) + actual = await get_accessibility_tree(mock_stagehand_page, mock_stagehand_logger) + + assert ( +"""[2] RootWebArea + [8] generic + [9] heading: Range Input + [10] generic + [11] slider: Volume value=5 (valuemin=0, valuemax=11) + [15] LabelText + [17] StaticText: Volume +""" + ) == actual["simplified"] + assert [] == actual["iframes"] + assert {"2": "https://example.com/input-range.html"} == actual["idToUrl"] + + async def test_list_links(self, mock_stagehand_page: StagehandPage, mock_stagehand_logger:StagehandLogger, mock_send_cdp, load_ax_tree): + mock_send_cdp( + ax_tree=load_ax_tree("list_links.json"), + backend_nodes={}, + tag_names={}, + ) + actual = await get_accessibility_tree(mock_stagehand_page, mock_stagehand_logger) + + assert ( +"""[2] RootWebArea + [8] generic + [9] heading: List Links + [10] list + [11] listitem + [20] ListMarker: • + [12] link: Google + [13] listitem + [22] ListMarker: • + [14] link: DuckDuckGo + [15] listitem + [24] ListMarker: • + [16] link: Bing + [17] listitem + [26] ListMarker: • + [18] link: Brave +""" + ) == actual["simplified"] + assert [] == actual["iframes"] + assert { + '12': 'https://www.google.com/', + '14': 'https://duckduckgo.com/', + '16': 'https://www.bing.com/', + '18': 'https://search.brave.com/', + '2': 'https://example.com/list_links.html' + } == actual["idToUrl"]