Support UI Automation custom annotations such as notes in MS Excel (n…

…vaccess#12861)

The Microsoft UI Automation accessibility API has a concept of annotations, which are essentially a way of attaching extra meta information (or annotations) to content. E.g. comments. An annotation is made up of both a known type ID E.g. AnnotationType_Comment, and an object (an extra UI automation element containing properties such as the author, date etc). NVDA already supports standard UI Automation annotations.
In Windows 11, UI Automation has been extended to support custom annotations. these are annotations with an application-defined type ID. E.g. an Excel note, or an MS word bookmark.
In order for these type IDs to be agreed upon by both the application and assistive technology at runtime, a mechanism very similar to UI Automation custom property registration was introduced to UI automation for registering custom annotation types, exposed via the Windows.UI.UIAutomation.Core.CoreAutomationRegistrar winRT interface.

Description of how this pull request fixes the issue:
• implemented a new registerUIAAnnotationType function in nvdaHelperLocal that uses the Windows.UI.UIAutomation.Core.CoreAutomationRegistrar winRT interface to register an annotation type GUID, resulting in a new annotation type ID that can be used like any other standard UI automation annotation type ID.
• Implemented infrastructure in NVDA to aide in registering UI Automation annotation types, pretty much identical to the UI Automation custom property registration infrastructure.
• Added support for detecting notes in MS Excel, which are exposed as a custom annotation on cells.
• Added support for detecting bookmarks in Microsoft Word documents in both speech and braille.
• Added support for detecting draft comments and resolved comments in Microsoft Word in both speech and braille.

nvdaHelper/local/UIAUtils.cpp

@@ -18,3 +18,13 @@ PROPERTYID registerUIAProperty(GUID* guid, LPCWSTR programmaticName, UIAutomatio
 	registrar->Release();
 	return propertyId;
 }
+
+int registerUIAAnnotationType(GUID* guid) {
+	if(!guid) {
+		LOG_DEBUGWARNING(L"NULL GUID given");
+		return 0;
+	}
+	winrt::Windows::UI::UIAutomation::Core::CoreAutomationRegistrar registrar {};
+	auto res = registrar.RegisterAnnotationType(*guid);
+	return res.LocalId;
+}

nvdaHelper/local/UIAUtils.h

@@ -1,9 +1,15 @@
 #ifndef NVDAHELPERLOCAL_UIAUTILS_H
 #define NVDAHELPERLOCAL_UIAUTILS_H
 
+// The following header included to allow winrt::guid to be converted to GUID
+#include <unknwn.h>
+
+#include <winrt/windows.ui.uiautomation.core.h>
 #include <uiAutomationCore.h>
 
 PROPERTYID registerUIAProperty(GUID* guid, LPCWSTR programmaticName, UIAutomationType propertyType);
+int registerUIAAnnotationType(GUID* guid);
+
 
 #endif
 

nvdaHelper/local/nvdaHelperLocal.def

@@ -55,6 +55,7 @@ EXPORTS
 	calculateCharacterOffsets
 	findWindowWithClassInThread
 	registerUIAProperty
+	registerUIAAnnotationType
 	dllImportTableHooks_hookSingle
 	dllImportTableHooks_unhookSingle
 	audioDucking_shouldDelay

readme.md

@@ -68,7 +68,7 @@ The following dependencies need to be installed on your system:
 				* Desktop development with C++
 			* Then in the Installation details tree view, under Desktop for C++, Optional, ensure the following are selected:
 				* MSVC v142 - VS 2019 C++ x64/x86 build tools
-				* Windows 10 SDK (10.0.19041.0)
+				* Windows 11 SDK (10.0.22000.0)
 				* C++ ATL for v142 build tools (x86 & x64)
 				* C++ Clang tools for Windows
 		* On the Individual components tab, ensure the following items are selected:

source/NVDAObjects/UIA/__init__.py

@@ -17,6 +17,7 @@
 import languageHandler
 import UIAHandler
 import _UIACustomProps
+import _UIACustomAnnotations
 import globalVars
 import eventHandler
 import controlTypes
@@ -159,7 +160,12 @@ def _getFormatFieldAtRange(self,textRange,formatConfig,ignoreMixedValues=False):
 		formatField=textInfos.FormatField()
 		if not isinstance(textRange,UIAHandler.IUIAutomationTextRange):
 			raise ValueError("%s is not a text range"%textRange)
-		fetchAnnotationTypes=formatConfig["reportSpellingErrors"] or formatConfig["reportComments"] or formatConfig["reportRevisions"]
+		fetchAnnotationTypes = (
+			formatConfig["reportSpellingErrors"]
+			or formatConfig["reportComments"]
+			or formatConfig["reportRevisions"]
+			or formatConfig["reportBookmarks"]
+		)
 		try:
 			textRange=textRange.QueryInterface(UIAHandler.IUIAutomationTextRange3)
 		except (COMError,AttributeError):
@@ -291,13 +297,22 @@ def _getFormatFieldAtRange(self,textRange,formatConfig,ignoreMixedValues=False):
 				if UIAHandler.AnnotationType_GrammarError in annotationTypes:
 					formatField["invalid-grammar"]=True
 			if formatConfig["reportComments"]:
-				if UIAHandler.AnnotationType_Comment in annotationTypes:
-					formatField["comment"]=True
+				cats = self.obj._UIACustomAnnotationTypes
+				if cats.microsoftWord_draftComment.id and cats.microsoftWord_draftComment.id in annotationTypes:
+					formatField["comment"] = textInfos.CommentType.DRAFT
+				elif cats.microsoftWord_resolvedComment.id and cats.microsoftWord_resolvedComment.id in annotationTypes:
+					formatField["comment"] = textInfos.CommentType.RESOLVED
+				elif UIAHandler.AnnotationType_Comment in annotationTypes:
+					formatField["comment"] = True
 			if formatConfig["reportRevisions"]:
 				if UIAHandler.AnnotationType_InsertionChange in annotationTypes:
 					formatField["revision-insertion"]=True
 				elif UIAHandler.AnnotationType_DeletionChange in annotationTypes:
 					formatField["revision-deletion"]=True
+			if formatConfig["reportBookmarks"]:
+				cats = self.obj._UIACustomAnnotationTypes
+				if cats.microsoftWord_bookmark.id and cats.microsoftWord_bookmark.id in annotationTypes:
+					formatField["bookmark"] = True
 		cultureVal=fetcher.getValue(UIAHandler.UIA_CultureAttributeId,ignoreMixedValues=ignoreMixedValues)
 		if cultureVal and isinstance(cultureVal,int):
 			try:
@@ -903,6 +918,7 @@ def updateSelection(self):
 
 class UIA(Window):
 	_UIACustomProps = _UIACustomProps.CustomPropertiesCommon.get()
+	_UIACustomAnnotationTypes = _UIACustomAnnotations.CustomAnnotationTypesCommon.get()
 
 	shouldAllowDuplicateUIAFocusEvent = False
 

source/NVDAObjects/UIA/excel.py

@@ -4,6 +4,8 @@
 # Copyright (C) 2018-2021 NV Access Limited, Leonard de Ruijter
 
 from typing import Optional, Tuple
+from comtypes import COMError
+import winVersion
 import UIAHandler
 import _UIAHandler
 import _UIAConstants
@@ -16,6 +18,9 @@
 from _UIACustomProps import (
 	CustomPropertyInfo,
 )
+from _UIACustomAnnotations import (
+	CustomAnnotationTypeInfo,
+)
 from comtypes import GUID
 from scriptHandler import script
 import ui
@@ -90,10 +95,39 @@ def __init__(self):
 		)
 
 
+class ExcelCustomAnnotationTypes:
+	""" UIA 'custom annotation types' specific to Excel.
+	Once registered, all subsequent registrations will return the same ID value.
+	This class should be used as a singleton via ExcelCustomAnnotationTypes.get()
+	to prevent unnecessary work by repeatedly interacting with UIA.
+	"""
+	#: Singleton instance
+	_instance: "Optional[ExcelCustomAnnotationTypes]" = None
+
+	@classmethod
+	def get(cls) -> "ExcelCustomAnnotationTypes":
+		"""Get the singleton instance or initialise it.
+		"""
+		if cls._instance is None:
+			cls._instance = cls()
+		return cls._instance
+
+	def __init__(self):
+		#  Available custom Annotations list at https://docs.microsoft.com/en-us/office/uia/excel/excelannotations
+		# Note annotation:
+		# Represents an old-style comment (now known as a note)
+		# which contains non-threaded plain text content.
+		self.note = CustomAnnotationTypeInfo(
+			guid=GUID("{4E863D9A-F502-4A67-808F-9E711702D05E}"),
+		)
+
+
 class ExcelObject(UIA):
 	"""Common base class for all Excel UIA objects
 	"""
 	_UIAExcelCustomProps = ExcelCustomProperties.get()
+	_UIAExcelCustomAnnotationTypes = ExcelCustomAnnotationTypes.get()
+
 
 
 class ExcelCell(ExcelObject):
@@ -367,6 +401,15 @@ def _get_states(self):
 			states.add(controlTypes.State.HASFORMULA)
 		if self._getUIACacheablePropertyValue(self._UIAExcelCustomProps.hasDataValidationDropdown.id):
 			states.add(controlTypes.State.HASPOPUP)
+		if winVersion.getWinVer() >= winVersion.WIN11:
+			try:
+				annotationTypes = self._getUIACacheablePropertyValue(UIAHandler.UIA_AnnotationTypesPropertyId)
+			except COMError:
+				# annotationTypes cannot be fetched on older Operating Systems such as Windows 7.
+				annotationTypes = None
+			if annotationTypes:
+				if self._UIAExcelCustomAnnotationTypes.note.id in annotationTypes:
+					states.add(controlTypes.State.HASNOTE)
 		return states
 
 	def _get_cellCoordsText(self):
@@ -421,28 +464,39 @@ def _get_cellCoordsText(self):
 		description=_("Reports the note or comment thread on the current cell"),
 		gesture="kb:NVDA+alt+c")
 	def script_reportComment(self, gesture):
+		if winVersion.getWinVer() >= winVersion.WIN11:
+			noteElement = self.UIAAnnotationObjects.get(self._UIAExcelCustomAnnotationTypes.note.id)
+			if noteElement:
+				name = noteElement.CurrentName
+				desc = noteElement.GetCurrentPropertyValue(UIAHandler.UIA_FullDescriptionPropertyId)
+				# Translators: a note on a cell in Microsoft excel.
+				text = _("{name}: {desc}").format(name=name, desc=desc)
+				ui.message(text)
+			else:
+				# Translators: message when a cell in Excel contains no note
+				ui.message(_("No note on this cell"))
 		commentsElement = self.UIAAnnotationObjects.get(UIAHandler.AnnotationType_Comment)
 		if commentsElement:
 			comment = commentsElement.GetCurrentPropertyValue(UIAHandler.UIA_FullDescriptionPropertyId)
 			author = commentsElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationAuthorPropertyId)
 			numReplies = commentsElement.GetCurrentPropertyValue(self._UIAExcelCustomProps.commentReplyCount.id)
 			if numReplies == 0:
 				# Translators: a comment on a cell in Microsoft excel.
-				text = _("{comment}  by {author}").format(
+				text = _("Comment thread: {comment}  by {author}").format(
 					comment=comment,
 					author=author
 				)
 			else:
 				# Translators: a comment on a cell in Microsoft excel.
-				text = _("{comment}  by {author} with {numReplies} replies").format(
+				text = _("Comment thread: {comment}  by {author} with {numReplies} replies").format(
 					comment=comment,
 					author=author,
 					numReplies=numReplies
 				)
 			ui.message(text)
 		else:
-			# Translators: A message in Excel when there is no note
-			ui.message(_("No note or comment thread on this cell"))
+			# Translators: A message in Excel when there is no comment thread
+			ui.message(_("No comment thread on this cell"))
 
 
 class ExcelWorksheet(ExcelObject):

source/NVDAObjects/UIA/wordDocument.py

@@ -32,6 +32,7 @@
 #: the non-printable unicode character that represents the end of cell or end of row mark in Microsoft Word
 END_OF_ROW_MARK = '\x07'
 
+
 class ElementsListDialog(browseMode.ElementsListDialog):
 
 	ELEMENT_TYPES=(browseMode.ElementsListDialog.ELEMENT_TYPES[0],browseMode.ElementsListDialog.ELEMENT_TYPES[1],
@@ -80,7 +81,12 @@ def getCommentInfoFromPosition(position):
 		UIAElement=UIAElement.buildUpdatedCache(UIAHandler.handler.baseCacheRequest)
 		typeID = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationAnnotationTypeIdPropertyId)
 		# Use Annotation Type Comment if available
-		if typeID == UIAHandler.AnnotationType_Comment:
+		cats = position.obj._UIACustomAnnotationTypes
+		if (
+			typeID == UIAHandler.AnnotationType_Comment
+			or (typeID and typeID == cats.microsoftWord_draftComment)
+			or (typeID and typeID == cats.microsoftWord_resolvedComment)
+		):
 			comment = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_NamePropertyId)
 			author = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationAuthorPropertyId)
 			date = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationDateTimePropertyId)

source/_UIACustomAnnotations.py

@@ -0,0 +1,89 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2021 NV Access Limited
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+
+from dataclasses import (
+	dataclass,
+	field,
+)
+from typing import Optional
+
+from comtypes import (
+	GUID,
+	byref,
+)
+import winVersion
+
+
+"""
+This module provides helpers and a common format to define UIA custom annotation types.
+The common custom annotation types are defined here.
+Custom annotation types specific to an application should be defined within a NVDAObjects/UIA
+submodule specific to that application, E.G. 'NVDAObjects/UIA/excel.py'
+
+UIA originally had hard coded 'static' ID's for annotation types.
+For an example see 'AnnotationType_SpellingError' in
+`source/comInterfaces/_944DE083_8FB8_45CF_BCB7_C477ACB2F897_0_1_0.py`
+imported via `UIAutomationClient.py`.
+When a new annotation type was added the UIA spec had to be updated.
+Now a mechanism is in place to allow applications to register "custom annotation types".
+This relies on both the UIA server application and the UIA client application sharing a known
+GUID for the annotation type.
+"""
+
+
+@dataclass
+class CustomAnnotationTypeInfo:
+	"""Holds information about a CustomAnnotationType
+	This makes it easy to define custom annotation types to be loaded.
+	"""
+	guid: GUID
+	id: int = field(init=False)
+
+	def __post_init__(self) -> None:
+		""" The id field must be initialised at runtime.
+		A GUID uniquely identifies a custom annotation, but the UIA system relies on integer IDs.
+		Any application (clients or providers) can register a custom annotation type, subsequent applications
+		will get the same id for a given GUID.
+		Registering custom annotations is only supported on Windows 11 and above.
+		For any lesser version, id will be 0.
+
+		"""
+		if winVersion.getWinVer() >= winVersion.WIN11:
+			import NVDAHelper
+			self.id = NVDAHelper.localLib.registerUIAAnnotationType(
+				byref(self.guid),
+			)
+		else:
+			self.id = 0
+
+
+class CustomAnnotationTypesCommon:
+	"""UIA 'custom annotation types' common to all applications.
+	Once registered, all subsequent registrations will return the same ID value.
+	This class should be used as a singleton via CustomAnnotationTypesCommon.get()
+	to prevent unnecessary work by repeatedly interacting with UIA.
+	"""
+	#: Singleton instance
+	_instance: "Optional[CustomAnnotationTypesCommon]" = None
+
+	@classmethod
+	def get(cls) -> "CustomAnnotationTypesCommon":
+		"""Get the singleton instance or initialise it.
+		"""
+		if cls._instance is None:
+			cls._instance = cls()
+		return cls._instance
+
+	def __init__(self):
+		# Registration of Custom annotation types used across multiple applications or frameworks should go here.
+		self.microsoftWord_resolvedComment = CustomAnnotationTypeInfo(
+			guid=GUID("{A015030C-5B44-4EAC-B0CC-21BA35DE6D07}"),
+		)
+		self.microsoftWord_draftComment = CustomAnnotationTypeInfo(
+			guid=GUID("{26BAEBC6-591E-4116-BBCF-E9A7996CD169}"),
+		)
+		self.microsoftWord_bookmark = CustomAnnotationTypeInfo(
+			guid=GUID("{25330951-A372-4DB9-A88A-85137AD008D2}"),
+		)

source/braille.py

@@ -834,6 +834,29 @@ def getFormatFieldBraille(field, fieldCache, isAtStart, formatConfig):
 		oldLink=fieldCache.get("link")
 		if link and link != oldLink:
 			textList.append(roleLabels[controlTypes.Role.LINK])
+	if formatConfig["reportComments"]:
+		comment = field.get("comment")
+		oldComment = fieldCache.get("comment") if fieldCache is not None else None
+		if (comment or oldComment is not None) and comment != oldComment:
+			if comment:
+				if comment is textInfos.CommentType.DRAFT:
+					# Translators: Brailled when text contains a draft comment.
+					text = _("drft cmnt")
+				elif comment is textInfos.CommentType.RESOLVED:
+					# Translators: Brailled when text contains a resolved comment.
+					text = _("rslvd cmnt")
+				else:  # generic
+					# Translators: Brailled when text contains a generic comment.
+					text = _("cmnt")
+				textList.append(text)
+	if formatConfig["reportBookmarks"]:
+		bookmark = field.get("bookmark")
+		oldBookmark = fieldCache.get("bookmark") if fieldCache is not None else None
+		if (bookmark or oldBookmark is not None) and bookmark != oldBookmark:
+			if bookmark:
+				# Translators: brailled when text contains a bookmark
+				text = _("bkmk")
+				textList.append(text)
 	fieldCache.clear()
 	fieldCache.update(field)
 	return TEXT_SEPARATOR.join([x for x in textList if x])

source/config/configSpec.py

@@ -203,6 +203,7 @@
 	reportLinks = boolean(default=true)
 	reportGraphics = boolean(default=True)
 	reportComments = boolean(default=true)
+	reportBookmarks = boolean(default=true)
 	reportLists = boolean(default=true)
 	reportHeadings = boolean(default=true)
 	reportBlockQuotes = boolean(default=true)

source/controlTypes/state.py

@@ -71,6 +71,7 @@ def negativeDisplayString(self) -> str:
 	OVERFLOWING = 0x10000000000
 	UNLOCKED = 0x20000000000
 	HAS_ARIA_DETAILS = 0x40000000000
+	HASNOTE = 0x80000000000
 
 
 STATES_SORTED = frozenset([State.SORTED, State.SORTED_ASCENDING, State.SORTED_DESCENDING])
@@ -160,6 +161,8 @@ def negativeDisplayString(self) -> str:
 	# Translators: a state that denotes that the object is unlocked (such as an unlocked cell in a protected
 	# Excel spreadsheet).
 	State.UNLOCKED: _("unlocked"),
+	# Translators: a state that denotes the existance of a note.
+	State.HASNOTE: _("has note"),
 }