Skip to content

Instantly share code, notes, and snippets.

@mahboud
Last active December 12, 2024 10:51
Show Gist options
  • Save mahboud/4e34f6b30b38be92212a691744ad82b8 to your computer and use it in GitHub Desktop.
Save mahboud/4e34f6b30b38be92212a691744ad82b8 to your computer and use it in GitHub Desktop.
Trying to get the selected text in the frontmost App using AXUIElement
func getAXSelectedText() -> String? {
requestAccessibilityPermissions()
if let text = searchInFocusedTextField() {
return text
}
return nil
}
func searchInFocusedTextField() -> String? {
guard let appElement = getFrontMostAppElement() else { return nil }
var focusedElement: CFTypeRef?
let focusedElementResult = AXUIElementCopyAttributeValue(appElement, kAXFocusedUIElementAttribute as CFString, &focusedElement)
guard focusedElementResult == .success, let element = focusedElement else {
print("searchForFocusedTextField: Failed to get the focused element - \(focusedElementResult.rawValue) looking for \(kAXFocusedUIElementAttribute)")
return nil
}
if let text = getTextFromTextElement(element as! AXUIElement) {
return text
}
return searchRecursivelyForTextElementWithSelectedText(element as! AXUIElement)
}
func searchRecursivelyForTextElementWithSelectedText(_ element: AXUIElement) -> String? {
var childrenRef: CFTypeRef?
if AXUIElementCopyAttributeValue(element, kAXChildrenAttribute /*kAXVisibleChildrenAttribute*/ as CFString, &childrenRef) == .success,
let children = childrenRef as? [AXUIElement] {
// The kAXVisibleChildrenAttribute doesn't work when some element is partially visible?
// kAXVisibleChildrenAttribute doesn't return an element that may be partially visible, it seems
print("***count of children: \(children.count)")
var childLimit = 100 // how many is too many?
for child in children {
if let text = getTextFromTextElement(child) {
return text
}
print("prepare for recursion")
// The focus element may just be a container for the selected text we
// are looking for. Recurse into the child's children
if let text = searchRecursivelyForTextElementWithSelectedText(child) {
return text
}
childLimit -= 1
if (childLimit <= 0) {
print("too many children")
break
}
}
} else {
print("no children")
}
return nil
}
func getTextFromTextElement(_ element: AXUIElement) -> String? {
var role: CFTypeRef?
if AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &role) == .success,
let roleString = role as? String {
print("possible find type - roleString: \(roleString)")
if roleString.contains("Text") || roleString == "AXWebArea" {
print("Found a possible element with text!")
if let selectedText = getSelectedTextOfElement(element) {
print("Selected text: \(selectedText) in \(roleString)")
print("Found a text field with selected text!")
return selectedText
}
}
}
return nil
}
func getSelectedTextOfElement(_ element: AXUIElement) -> String? {
var selectedText: CFTypeRef?
if AXUIElementCopyAttributeValue(element, kAXSelectedTextAttribute as CFString, &selectedText) == .success,
let selectedTextString = selectedText as? String, !selectedTextString.isEmpty {
print("Element has selected text: \(selectedTextString)")
return selectedTextString
} else {
var selectionRange: CFTypeRef?
let selectionRangeResult = AXUIElementCopyAttributeValue(element, "AXIntersectionWithSelectionRange" as CFString, &selectionRange)
if selectionRangeResult == .success, let axValue = selectionRange {
var range = CFRange()
if AXValueGetValue(axValue as! AXValue, AXValueType.cfRange, &range) {
print("selection range: \(range.length) range.location: \(range.location)")
var text: CFTypeRef?
AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &text)
if range.length >= 10,
let textString = text as? String, textString.count > 0 {
print("Static text: \(textString)")
print("***Found a static text field with selected text!")
return textString
}
} else {
print("Failed to convert AXValue to CFRange.")
}
} else {
print("Element does not have selected text.")
}
}
return nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment