Get reliable points from offsets in virtual buffers (PR #8479)

Fixes #6460

1. Added two attributes to virtual buffer text nodes.
   - ia2TextOffset: holds the start offset for a TextFieldNode within the IAccessibleText of its associated 
     object. This attribute is only added to text field nodes that originate from IAccessibleText.
   - strippedCharsFromStart: When characters are stripped from the start of a text node (#2963) this 
      attribute holds the amount of stripped chars from the start, in order for us to correct the 
      ia2TextOffset.

2. the Gecko virtual buffer text info now has its own _getPointFromOffset implementation that uses the new virtual buffer attributes.

3. VirtualBufferTextInfo.getTextWithFields is now split up into a helper function _getFieldsInRange, similar to DisplayModelTextInfo. This allows for easy retrieval of control/format fields in a particular range, regardless where the text info is positioned.

4. pointFromOffset in the adobe acrobat virtual buffer now uses the _indexINParent attribute. It seems Adobe Acrobat has separate dom nodes for every word, so pointFromOffset now gets the center point of the word, which is far better than it was before.

nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp

@@ -779,6 +779,10 @@ VBufStorage_fieldNode_t* GeckoVBufBackend_t::fillVBuf(IAccessible2* pacc,
 					// Add the chunk to the buffer.
 					if(tempNode=buffer->addTextFieldNode(parentNode,previousNode,wstring(IA2Text+chunkStart,i-chunkStart))) {
 						previousNode=tempNode;
+						// Add the IA2Text start offset as an attribute on the node.
+						s << chunkStart;
+						previousNode->addAttribute(L"ia2TextStartOffset", s.str());
+						s.str(L"");
 						// Add text attributes.
 						for(map<wstring,wstring>::const_iterator it=textAttribs.begin();it!=textAttribs.end();++it)
 							previousNode->addAttribute(it->first,it->second);

nvdaHelper/vbufBase/storage.cpp

@@ -579,6 +579,13 @@ VBufStorage_textFieldNode_t*  VBufStorage_buffer_t::addTextFieldNode(VBufStorage
 		delete textFieldNode;
 		return NULL;
 	}
+	if (needsStrip && subStart>0) {
+		// There are characters stripped from the start of the text.
+		// We save the number of these characters in an attribute, required for calculating offsets in IA2Text.
+		wostringstream s;
+		s << subStart;
+		textFieldNode->addAttribute(L"strippedCharsFromStart", s.str());
+	}
 	return textFieldNode;
 }
 

source/NVDAObjects/IAccessible/__init__.py

@@ -120,14 +120,18 @@ def _getOffsetFromPoint(self,x,y):
 		else:
 			raise NotImplementedError
 
-	def _getPointFromOffset(self,offset):
+	@classmethod
+	def _getPointFromOffsetInObject(cls,obj,offset):
 		try:
-			res=self.obj.IAccessibleTextObject.characterExtents(offset,IAccessibleHandler.IA2_COORDTYPE_SCREEN_RELATIVE)
+			res=RectLTWH(*obj.IAccessibleTextObject.characterExtents(offset,IAccessibleHandler.IA2_COORDTYPE_SCREEN_RELATIVE))
 		except COMError:
 			raise NotImplementedError
-		point=textInfos.Point(res[0]+(res[2]/2),res[1]+(res[3]/2))
+		point=textInfos.Point(*res.center)
 		return point
 
+	def _getPointFromOffset(self,offset):
+		return self._getPointFromOffsetInObject(self.obj, offset)
+
 	def _get_unit_mouseChunk(self):
 		return "mouseChunk"
 

source/virtualBuffers/__init__.py

@@ -135,6 +135,7 @@ class VirtualBufferTextInfo(browseMode.BrowseModeDocumentTextInfo,textInfos.offs
 	allowMoveToOffsetPastEnd=False #: no need for end insertion point as vbuf is not editable. 
 
 	UNIT_CONTROLFIELD = "controlField"
+	UNIT_FORMATFIELD = "formatField"
 
 	def _getControlFieldAttribs(self,  docHandle, id):
 		info = self.copy()
@@ -155,6 +156,8 @@ def _getFieldIdentifierFromOffset(self, offset):
 		ID = ctypes.c_int()
 		node=VBufRemote_nodeHandle_t()
 		NVDAHelper.localLib.VBuf_locateControlFieldNodeAtOffset(self.obj.VBufHandle, offset, ctypes.byref(startOffset), ctypes.byref(endOffset), ctypes.byref(docHandle), ctypes.byref(ID),ctypes.byref(node))
+		if not any((docHandle.value, ID.value)):
+			raise LookupError("Neither docHandle nor ID found for offset %d" % offset)
 		return docHandle.value, ID.value
 
 	def _getOffsetsFromFieldIdentifier(self, docHandle, ID):
@@ -169,6 +172,8 @@ def _getOffsetsFromFieldIdentifier(self, docHandle, ID):
 
 	def _getPointFromOffset(self,offset):
 		o = self._getNVDAObjectFromOffset(offset)
+		if not o.location:
+			raise LookupError
 		left, top, width, height = o.location
 		return textInfos.Point(left + width / 2, top + height / 2)
 
@@ -252,11 +257,7 @@ def _getPlaceholderAttribute(self, attrs, placeholderAttrsKey):
 					return placeholder
 		return None
 
-	def getTextWithFields(self,formatConfig=None):
-		start=self._startOffset
-		end=self._endOffset
-		if start==end:
-			return ""
+	def _getFieldsInRange(self,start,end):
 		text=NVDAHelper.VBuf_getTextInRange(self.obj.VBufHandle,start,end,True)
 		if not text:
 			return ""
@@ -270,6 +271,13 @@ def getTextWithFields(self,formatConfig=None):
 					commandList[index].field=self._normalizeFormatField(field)
 		return commandList
 
+	def getTextWithFields(self,formatConfig=None):
+		start=self._startOffset
+		end=self._endOffset
+		if start==end:
+			return ""
+		return self._getFieldsInRange(start,end)
+
 	def _getWordOffsets(self,offset):
 		#Use VBuf_getBufferLineOffsets with out screen layout to find out the range of the current field
 		lineStart=ctypes.c_int()
@@ -334,6 +342,10 @@ def _normalizeControlField(self,attrs):
 		return attrs
 
 	def _normalizeFormatField(self, attrs):
+		strippedCharsFromStart = attrs.get("strippedCharsFromStart")
+		if strippedCharsFromStart is not None:
+			assert strippedCharsFromStart.isdigit(), "strippedCharsFromStart isn't a digit, %r" % strippedCharsFromStart
+			attrs["strippedCharsFromStart"] = int(strippedCharsFromStart)
 		return attrs
 
 	def _getLineNumFromOffset(self, offset):
@@ -351,6 +363,12 @@ def _getUnitOffsets(self, unit, offset):
 			node=VBufRemote_nodeHandle_t()
 			NVDAHelper.localLib.VBuf_locateControlFieldNodeAtOffset(self.obj.VBufHandle,offset,ctypes.byref(startOffset),ctypes.byref(endOffset),ctypes.byref(docHandle),ctypes.byref(ID),ctypes.byref(node))
 			return startOffset.value,endOffset.value
+		elif unit == self.UNIT_FORMATFIELD:
+			startOffset=ctypes.c_int()
+			endOffset=ctypes.c_int()
+			node=VBufRemote_nodeHandle_t()
+			NVDAHelper.localLib.VBuf_locateTextFieldNodeAtOffset(self.obj.VBufHandle,offset,ctypes.byref(startOffset),ctypes.byref(endOffset),ctypes.byref(node))
+			return startOffset.value,endOffset.value
 		return super(VirtualBufferTextInfo, self)._getUnitOffsets(unit, offset)
 
 	def _get_clipboardText(self):

source/virtualBuffers/adobeAcrobat.py

@@ -18,6 +18,29 @@
 
 class AdobeAcrobat_TextInfo(VirtualBufferTextInfo):
 
+	def _getPointFromOffset(self,offset):
+		formatFieldStart, formatFieldEnd = self._getUnitOffsets(self.UNIT_FORMATFIELD, offset)
+		# The format field starts at the first character.
+		for field in reversed(self._getFieldsInRange(formatFieldStart, formatFieldStart+1)):
+			if not (isinstance(field, textInfos.FieldCommand) and field.command == "formatChange"):
+				# This is no format field.
+				continue
+			attrs = field.field
+			indexInParent = attrs.get("_indexInParent")
+			if indexInParent is None:
+				continue
+			try:
+				obj = self._getNVDAObjectFromOffset(offset).getChild(indexInParent)
+			except IndexError:
+				obj = None
+			if not obj:
+				continue
+			if not obj.location:
+				# Older versions of Adobe Reader have per word objects, but they don't expose a location
+				break
+			return textInfos.Point(*obj.location.center)
+		return super(AdobeAcrobat_TextInfo, self)._getPointFromOffset(offset)
+
 	def _normalizeControlField(self,attrs):
 		stdName = attrs.get("acrobat::stdname", "")
 		try:
@@ -51,6 +74,10 @@ def _normalizeFormatField(self, attrs):
 			attrs["language"] = languageHandler.normalizeLanguage(attrs["language"])
 		except KeyError:
 			pass
+		try:
+			attrs["_indexInParent"] = int(attrs["_indexInParent"])
+		except KeyError:
+			pass
 		return attrs
 
 class AdobeAcrobat(VirtualBuffer):

source/virtualBuffers/gecko_ia2.py

@@ -18,10 +18,30 @@
 from comtypes import COMError
 import aria
 import config
-from NVDAObjects.IAccessible import normalizeIA2TextFormatField
+from NVDAObjects.IAccessible import normalizeIA2TextFormatField, IA2TextTextInfo
 
 class Gecko_ia2_TextInfo(VirtualBufferTextInfo):
 
+	def _getPointFromOffset(self,offset):
+		formatFieldStart, formatFieldEnd = self._getUnitOffsets(self.UNIT_FORMATFIELD, offset)
+		# The format field starts at the first character.
+		for field in reversed(self._getFieldsInRange(formatFieldStart, formatFieldStart+1)):
+			if not (isinstance(field, textInfos.FieldCommand) and field.command == "formatChange"):
+				# This is no format field.
+				continue
+			attrs = field.field
+			ia2TextStartOffset = attrs.get("ia2TextStartOffset")
+			if ia2TextStartOffset is None:
+				# No ia2TextStartOffset specified, most likely a format field that doesn't originate from IA2Text.
+				continue
+			ia2TextStartOffset += attrs.get("strippedCharsFromStart", 0)
+			relOffset = offset - formatFieldStart + ia2TextStartOffset
+			obj = self._getNVDAObjectFromOffset(offset)
+			if not hasattr(obj, "IAccessibleTextObject"):
+				raise LookupError("Object doesn't have an IAccessibleTextObject")
+			return IA2TextTextInfo._getPointFromOffsetInObject(obj, relOffset)
+		return super(Gecko_ia2_TextInfo, self)._getPointFromOffset(offset)
+
 	def _normalizeControlField(self,attrs):
 		for attr in ("table-physicalrownumber","table-physicalcolumnnumber","table-physicalrowcount","table-physicalcolumncount"):
 			attrVal=attrs.get(attr)
@@ -84,7 +104,11 @@ def _normalizeControlField(self,attrs):
 
 	def _normalizeFormatField(self, attrs):
 		normalizeIA2TextFormatField(attrs)
-		return attrs
+		ia2TextStartOffset = attrs.get("ia2TextStartOffset")
+		if ia2TextStartOffset is not None:
+			assert ia2TextStartOffset.isdigit(), "ia2TextStartOffset isn't a digit, %r" % ia2TextStartOffset
+			attrs["ia2TextStartOffset"] = int(ia2TextStartOffset)
+		return super(Gecko_ia2_TextInfo,self)._normalizeFormatField(attrs)
 
 class Gecko_ia2(VirtualBuffer):
 

user_docs/en/changes.t2t

@@ -17,6 +17,7 @@ What's New in NVDA
 == Bug Fixes ==
 - In Outlook 2016/365, the category and flag status are reported for messages. (#8603)
 - When NVDA is set to languages such as Kirgyz, Mongolian or Macedonian, it no longer shows a dialog on start-up warning that the language is not supported by the Operating System. (#8064)
+- Moving the mouse to the navigator object will now much more accurately move the mouse to the browse mode position in Mozilla Firefox, Google Chrome and Acrobat Reader DC. (#6460)
 
 
 == Changes for Developers ==