Last active
December 12, 2024 10:51
-
-
Save mahboud/4e34f6b30b38be92212a691744ad82b8 to your computer and use it in GitHub Desktop.
Trying to get the selected text in the frontmost App using AXUIElement
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
func getAXSelectedText() -> String? { | |
requestAccessibilityPermissions() | |
if let text = searchInFocusedTextField() { | |
return text | |
} | |
return nil | |
} | |
func searchInFocusedTextField() -> String? { | |
guard let appElement = getFrontMostAppElement() else { return nil } | |
var focusedElement: CFTypeRef? | |
let focusedElementResult = AXUIElementCopyAttributeValue(appElement, kAXFocusedUIElementAttribute as CFString, &focusedElement) | |
guard focusedElementResult == .success, let element = focusedElement else { | |
print("searchForFocusedTextField: Failed to get the focused element - \(focusedElementResult.rawValue) looking for \(kAXFocusedUIElementAttribute)") | |
return nil | |
} | |
if let text = getTextFromTextElement(element as! AXUIElement) { | |
return text | |
} | |
return searchRecursivelyForTextElementWithSelectedText(element as! AXUIElement) | |
} | |
func searchRecursivelyForTextElementWithSelectedText(_ element: AXUIElement) -> String? { | |
var childrenRef: CFTypeRef? | |
if AXUIElementCopyAttributeValue(element, kAXChildrenAttribute /*kAXVisibleChildrenAttribute*/ as CFString, &childrenRef) == .success, | |
let children = childrenRef as? [AXUIElement] { | |
// The kAXVisibleChildrenAttribute doesn't work when some element is partially visible? | |
// kAXVisibleChildrenAttribute doesn't return an element that may be partially visible, it seems | |
print("***count of children: \(children.count)") | |
var childLimit = 100 // how many is too many? | |
for child in children { | |
if let text = getTextFromTextElement(child) { | |
return text | |
} | |
print("prepare for recursion") | |
// The focus element may just be a container for the selected text we | |
// are looking for. Recurse into the child's children | |
if let text = searchRecursivelyForTextElementWithSelectedText(child) { | |
return text | |
} | |
childLimit -= 1 | |
if (childLimit <= 0) { | |
print("too many children") | |
break | |
} | |
} | |
} else { | |
print("no children") | |
} | |
return nil | |
} | |
func getTextFromTextElement(_ element: AXUIElement) -> String? { | |
var role: CFTypeRef? | |
if AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &role) == .success, | |
let roleString = role as? String { | |
print("possible find type - roleString: \(roleString)") | |
if roleString.contains("Text") || roleString == "AXWebArea" { | |
print("Found a possible element with text!") | |
if let selectedText = getSelectedTextOfElement(element) { | |
print("Selected text: \(selectedText) in \(roleString)") | |
print("Found a text field with selected text!") | |
return selectedText | |
} | |
} | |
} | |
return nil | |
} | |
func getSelectedTextOfElement(_ element: AXUIElement) -> String? { | |
var selectedText: CFTypeRef? | |
if AXUIElementCopyAttributeValue(element, kAXSelectedTextAttribute as CFString, &selectedText) == .success, | |
let selectedTextString = selectedText as? String, !selectedTextString.isEmpty { | |
print("Element has selected text: \(selectedTextString)") | |
return selectedTextString | |
} else { | |
var selectionRange: CFTypeRef? | |
let selectionRangeResult = AXUIElementCopyAttributeValue(element, "AXIntersectionWithSelectionRange" as CFString, &selectionRange) | |
if selectionRangeResult == .success, let axValue = selectionRange { | |
var range = CFRange() | |
if AXValueGetValue(axValue as! AXValue, AXValueType.cfRange, &range) { | |
print("selection range: \(range.length) range.location: \(range.location)") | |
var text: CFTypeRef? | |
AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &text) | |
if range.length >= 10, | |
let textString = text as? String, textString.count > 0 { | |
print("Static text: \(textString)") | |
print("***Found a static text field with selected text!") | |
return textString | |
} | |
} else { | |
print("Failed to convert AXValue to CFRange.") | |
} | |
} else { | |
print("Element does not have selected text.") | |
} | |
} | |
return nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment