Auto merge of #843 - micbou:improve-clang-include-completion, r=micbou

[READY] Improve completion of include statements in C-family languages

This PR fixes the issue where completion is interrupted after inserting a non-identifier character in include statements. See issues ycm-core/YouCompleteMe#281 and ycm-core/YouCompleteMe#1553.

This is done by moving the include completion logic from the filename completer to the Clang one and by setting the start column to the right position thanks to PR #681. A benefit of moving the logic to the Clang completer is that `<C-Space>` now works in include statements.

Here's a demo before the changes:

![include-statement-before](https://user-images.githubusercontent.com/10026824/30808129-1c03f634-a1fd-11e7-8de3-0fcc84424d89.gif)

and after:

![include-statement-after](https://user-images.githubusercontent.com/10026824/30808134-1e8446e8-a1fd-11e7-9cbe-7bf9b7583fb2.gif)

You'll notice that no error is shown to the user after inserting the dot.

Fixes ycm-core/YouCompleteMe#281.
Fixes ycm-core/YouCompleteMe#1553.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/ycmd/843)
<!-- Reviewable:end -->

ycmd/completers/completer_utils.py

@@ -212,38 +212,6 @@ def FilterAndSortCandidatesWrap( candidates, sort_property, query,
     DEFAULT_FILETYPE_TRIGGERS )
 
 
-INCLUDE_REGEX = re.compile( '\s*#\s*(?:include|import)\s*("|<)' )
-
-
-def AtIncludeStatementStart( line ):
-  match = INCLUDE_REGEX.match( line )
-  if not match:
-    return False
-  # Check if the whole string matches the regex
-  return match.end() == len( line )
-
-
-def GetIncludeStatementValue( line, check_closing = True ):
-  """Returns include statement value and boolean indicating whether
-     include statement is quoted.
-     If check_closing is True the string is scanned for statement closing
-     character (" or >) and substring between opening and closing characters is
-     returned. The whole string after opening character is returned otherwise"""
-  match = INCLUDE_REGEX.match( line )
-  include_value = None
-  quoted_include = False
-  if match:
-    quoted_include = ( match.group( 1 ) == '"' )
-    if not check_closing:
-      include_value = line[ match.end(): ]
-    else:
-      close_char = '"' if quoted_include else '>'
-      close_char_pos = line.find( close_char, match.end() )
-      if close_char_pos != -1:
-        include_value = line[ match.end() : close_char_pos ]
-  return include_value, quoted_include
-
-
 def GetFileContents( request_data, filename ):
   """Returns the contents of the absolute path |filename| as a unicode
   string. If the file contents exist in |request_data| (i.e. it is open and

ycmd/completers/cpp/clang_completer.py

@@ -25,6 +25,7 @@
 
 from collections import defaultdict
 from future.utils import iteritems
+import logging
 import os.path
 import re
 import textwrap
@@ -34,10 +35,12 @@
 from ycmd import responses
 from ycmd.utils import ToCppStringCompatible, ToUnicode
 from ycmd.completers.completer import Completer
-from ycmd.completers.completer_utils import GetIncludeStatementValue
 from ycmd.completers.cpp.flags import ( Flags, PrepareFlagsForClang,
-                                        NoCompilationDatabase )
+                                        NoCompilationDatabase,
+                                        UserIncludePaths )
 from ycmd.completers.cpp.ephemeral_values_set import EphemeralValuesSet
+from ycmd.completers.general.filename_completer import (
+    GenerateCandidatesForPaths )
 from ycmd.responses import NoExtraConfDetected, UnknownExtraConf
 
 CLANG_FILETYPES = set( [ 'c', 'cpp', 'objc', 'objcpp' ] )
@@ -49,6 +52,7 @@
 PRAGMA_DIAG_TEXT_TO_IGNORE = '#pragma once in main file'
 TOO_MANY_ERRORS_DIAG_TEXT_TO_IGNORE = 'too many errors emitted, stopping now'
 NO_DOCUMENTATION_MESSAGE = 'No documentation available for current context'
+INCLUDE_REGEX = re.compile( '(\s*#\s*(?:include|import)\s*)(:?"[^"]*|<[^>]*)' )
 
 
 class ClangCompleter( Completer ):
@@ -60,6 +64,7 @@ def __init__( self, user_options ):
     self._flags = Flags()
     self._diagnostic_store = None
     self._files_being_compiled = EphemeralValuesSet()
+    self._logger = logging.getLogger( __name__ )
 
 
   def SupportedFiletypes( self ):
@@ -85,11 +90,62 @@ def GetUnsavedFilesVector( self, request_data ):
     return files
 
 
+  def ShouldCompleteIncludeStatement( self, request_data ):
+    column_codepoint = request_data[ 'column_codepoint' ] - 1
+    current_line = request_data[ 'line_value' ]
+    return INCLUDE_REGEX.match( current_line[ : column_codepoint ] )
+
+
+  def ShouldUseNowInner( self, request_data ):
+    if self.ShouldCompleteIncludeStatement( request_data ):
+      return True
+    return super( ClangCompleter, self ).ShouldUseNowInner( request_data )
+
+
+  def GetIncludePaths( self, request_data ):
+    column_codepoint = request_data[ 'column_codepoint' ] - 1
+    current_line = request_data[ 'line_value' ]
+    line = current_line[ : column_codepoint ]
+    path_dir, quoted_include, start_codepoint = (
+        GetIncompleteIncludeValue( line ) )
+    if start_codepoint is None:
+      return None
+
+    request_data[ 'start_codepoint' ] = start_codepoint
+
+    # We do what GCC does for <> versus "":
+    # http://gcc.gnu.org/onlinedocs/cpp/Include-Syntax.html
+    flags = self._FlagsForRequest( request_data )
+    filepath = request_data[ 'filepath' ]
+    quoted_include_paths, include_paths = UserIncludePaths( flags, filepath )
+    if quoted_include:
+      include_paths.extend( quoted_include_paths )
+
+    paths = []
+    for include_path in include_paths:
+      unicode_path = ToUnicode( os.path.join( include_path, path_dir ) )
+      try:
+        # We need to pass a unicode string to get unicode strings out of
+        # listdir.
+        relative_paths = os.listdir( unicode_path )
+      except Exception:
+        self._logger.exception( 'Error while listing %s folder.', unicode_path )
+        relative_paths = []
+
+      paths.extend( os.path.join( include_path, path_dir, relative_path ) for
+                    relative_path in relative_paths  )
+    return paths
+
+
   def ComputeCandidatesInner( self, request_data ):
     filename = request_data[ 'filepath' ]
     if not filename:
       return
 
+    paths = self.GetIncludePaths( request_data )
+    if paths is not None:
+      return GenerateCandidatesForPaths( paths )
+
     if self._completer.UpdatingTranslationUnit(
         ToCppStringCompatible( filename ) ):
       raise RuntimeError( PARSING_FILE_MESSAGE )
@@ -220,18 +276,18 @@ def _GoToImprecise( self, request_data ):
 
   def _ResponseForInclude( self, request_data ):
     """Returns response for include file location if cursor is on the
-       include statement, None otherwise.
-       Throws RuntimeError if cursor is on include statement and corresponding
-       include file not found."""
+    include statement, None otherwise.
+    Throws RuntimeError if cursor is on include statement and corresponding
+    include file not found."""
     current_line = request_data[ 'line_value' ]
-    include_file_name, quoted_include = GetIncludeStatementValue( current_line )
+    include_file_name, quoted_include = GetFullIncludeValue( current_line )
     if not include_file_name:
       return None
 
+    flags = self._FlagsForRequest( request_data )
     current_file_path = request_data[ 'filepath' ]
-    client_data = request_data.get( 'extra_conf_data', None )
-    quoted_include_paths, include_paths = (
-            self._flags.UserIncludePaths( current_file_path, client_data ) )
+    quoted_include_paths, include_paths = UserIncludePaths( flags,
+                                                            current_file_path )
     if quoted_include:
       include_file_path = _GetAbsolutePath( include_file_name,
                                             quoted_include_paths )
@@ -429,10 +485,6 @@ def ClangAvailableForFiletypes( filetypes ):
   return any( [ filetype in CLANG_FILETYPES for filetype in filetypes ] )
 
 
-def InCFamilyFile( filetypes ):
-  return ClangAvailableForFiletypes( filetypes )
-
-
 def _FilterDiagnostics( diagnostics ):
   # Clang has an annoying warning that shows up when we try to compile header
   # files if the header has "#pragma once" inside it. The error is not
@@ -523,3 +575,48 @@ def _GetAbsolutePath( include_file_name, include_paths ):
     if os.path.isfile( include_file_path ):
       return include_file_path
   return None
+
+
+def GetIncompleteIncludeValue( line ):
+  """Returns the tuple |include_value|, |quoted_include|, and |start_codepoint|
+  where:
+  - |include_value| is the string starting from the opening quote or bracket of
+    the include statement in |line|. None if no include statement is found;
+  - |quoted_include| is True if the statement is a quoted include, False
+    otherwise;
+  - |start_column| is the 1-based column where the completion should start (i.e.
+    at the last path separator '/' or at the opening quote or bracket). None if
+    no include statement is matched."""
+  match = INCLUDE_REGEX.match( line )
+  if not match:
+    return None, False, None
+
+  include_start = match.end( 1 ) + 1
+  quoted_include = ( line[ include_start - 1 ] == '"' )
+  separator_char = '/'
+  separator_char_pos = line.rfind( separator_char, match.end( 1 ) )
+  if separator_char_pos == -1:
+    return '', quoted_include, include_start + 1
+  return ( line[ include_start : separator_char_pos + 1 ],
+           quoted_include,
+           separator_char_pos + 2 )
+
+
+
+def GetFullIncludeValue( line ):
+  """Returns the tuple |include_value| and |quoted_include| where:
+  - |include_value| is the whole string inside the quotes or brackets of the
+    include statement in |line|. None if no include statement is found;
+  - |quoted_include| is True if the statement is a quoted include, False
+    otherwise."""
+  match = INCLUDE_REGEX.match( line )
+  if not match:
+    return None, False
+
+  include_start = match.end( 1 ) + 1
+  quoted_include = ( line[ include_start - 1 ] == '"' )
+  close_char = '"' if quoted_include else '>'
+  close_char_pos = line.find( close_char, match.end() )
+  if close_char_pos == -1:
+    return None, quoted_include
+  return line[ include_start : close_char_pos ], quoted_include

ycmd/completers/cpp/flags.py

@@ -141,6 +141,7 @@ def FlagsForFile( self,
 
     if add_extra_clang_flags:
       flags += self.extra_clang_flags
+      flags = _AddMacIncludePaths( flags )
 
     sanitized_flags = PrepareFlagsForClang( flags,
                                             filename,
@@ -158,43 +159,6 @@ def _GetFlagsFromExtraConfOrDatabase( self, module, filename, client_data ):
     return _CallExtraConfFlagsForFile( module, filename, client_data )
 
 
-  def UserIncludePaths( self, filename, client_data ):
-    flags = [ ToUnicode( x ) for x in
-              self.FlagsForFile( filename, client_data = client_data ) ]
-
-    quoted_include_paths = [ os.path.dirname( filename ) ]
-    include_paths = []
-
-    if flags:
-      quote_flag = '-iquote'
-      path_flags = [ '-isystem', '-I' ]
-
-      try:
-        it = iter( flags )
-        for flag in it:
-          flag_len = len( flag )
-          if flag.startswith( quote_flag ):
-            quote_flag_len = len( quote_flag )
-            # Add next flag to the include paths if current flag equals to
-            # '-iquote', or add remaining string otherwise.
-            quoted_include_paths.append( next( it )
-                                         if flag_len == quote_flag_len
-                                         else flag[ quote_flag_len: ] )
-          else:
-            for path_flag in path_flags:
-              if flag.startswith( path_flag ):
-                path_flag_len = len( path_flag )
-                include_paths.append( next( it )
-                                      if flag_len == path_flag_len
-                                      else flag[ path_flag_len: ] )
-                break
-      except StopIteration:
-        pass
-
-    return ( [ x for x in quoted_include_paths if x ],
-             [ x for x in include_paths if x ] )
-
-
   def Clear( self ):
     self.flags_for_file.clear()
     self.compilation_database_dir_map.clear()
@@ -314,9 +278,6 @@ def PrepareFlagsForClang( flags, filename, add_extra_clang_flags = True ):
   flags = _RemoveXclangFlags( flags )
   flags = _RemoveUnusedFlags( flags, filename )
   if add_extra_clang_flags:
-    if OnMac() and not _SysRootSpecifedIn( flags ):
-      for path in _MacIncludePaths():
-        flags.extend( [ '-isystem', path ] )
     flags = _EnableTypoCorrection( flags )
 
   vector = ycm_core.StringVector()
@@ -519,9 +480,11 @@ def _LatestMacClangIncludes( toolchain ):
     )
 
 
-def _MacIncludePaths():
-  # This method exists for testing only
-  return MAC_INCLUDE_PATHS
+def _AddMacIncludePaths( flags ):
+  if OnMac() and not _SysRootSpecifedIn( flags ):
+    for path in MAC_INCLUDE_PATHS:
+      flags.extend( [ '-isystem', path ] )
+  return flags
 
 
 def _ExtraClangFlags():
@@ -624,3 +587,38 @@ def _GetCompilationInfoForFile( database, file_name, file_extension ):
   # No corresponding source file was found, so we can't generate any flags for
   # this source file.
   return None
+
+
+def UserIncludePaths( flags, filename ):
+  quoted_include_paths = [ os.path.dirname( filename ) ]
+  include_paths = []
+
+  if flags:
+    quote_flag = '-iquote'
+    path_flags = [ '-isystem', '-I' ]
+
+    try:
+      it = iter( flags )
+      for flag in it:
+        flag_len = len( flag )
+        if flag.startswith( quote_flag ):
+          quote_flag_len = len( quote_flag )
+          # Add next flag to the include paths if current flag equals to
+          # '-iquote', or add remaining string otherwise.
+          quoted_include_path = ( next( it ) if flag_len == quote_flag_len else
+                                  flag[ quote_flag_len: ] )
+          if quoted_include_path:
+            quoted_include_paths.append( ToUnicode( quoted_include_path ) )
+        else:
+          for path_flag in path_flags:
+            if flag.startswith( path_flag ):
+              path_flag_len = len( path_flag )
+              include_path = ( next( it ) if flag_len == path_flag_len else
+                               flag[ path_flag_len: ] )
+              if include_path:
+                include_paths.append( ToUnicode( include_path ) )
+              break
+    except StopIteration:
+      pass
+
+  return quoted_include_paths, include_paths