From 22abb751c162812be3324ce2e17c84fc1426416c Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Tue, 24 May 2016 00:05:01 +0100 Subject: [PATCH 1/9] Support for default extra conf file for trival projects --- ycmd/default_ycm_extra_conf.py | 381 +++++++++++++++++++++++++++++++++ ycmd/extra_conf_store.py | 14 +- 2 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 ycmd/default_ycm_extra_conf.py diff --git a/ycmd/default_ycm_extra_conf.py b/ycmd/default_ycm_extra_conf.py new file mode 100644 index 0000000000..180313cad5 --- /dev/null +++ b/ycmd/default_ycm_extra_conf.py @@ -0,0 +1,381 @@ +# This file is NOT licensed under the GPLv3, which is the license for the rest +# of ycmd. +# +# Here's the license text for this file: +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# For more information, please refer to + +import os + +from ycmd import utils +import ycm_core + +# Overview +# -------- +# +# This file provides an attempt to heuristically provide compiler flags for +# c-family languages. It works best when a compilation database is used. A +# compilation database can be generated by tools such as cmake, bear, etc. It +# contains the compiler flags (actually, a compiler invocation) for each source +# file in the project, generated by the make system. If no compilation database +# is used, an attempt is made to guess some flags. +# +# Finding the compilation database +# -------------------------------- +# +# While there is no standard, and using something like cmake typically involves +# an out-of-tree build, we attempt to find a compilation database by walking the +# directory hierarchy looking for a file named compile_commands.json, starting +# at the path of the file for which flags were requested (i.e. the file being +# edited). +# +# If a compilation database is found, it is cached against the directory it is +# found in, to prevent re-parsing the entire database for every file, while +# allowing a single instance ycmd to operate on multiple "projects" at once. +# +# Guessing flags +# -------------- +# +# The compilation database typically only contains flags (compiler invocations) +# for source files known to the make system and/or makefile generator, as these +# form the translation units for which calls to the compiler (typically with the +# -c flag) are actually made. It does not contain: +# - new files, not yet run through the make generator system +# - header files +# +# Additionally, not all projects will have a compilation database. While it is +# recommended to generate one (again, see 'bear' if you don't use cmake), we try +# to make certain toy or example projects work without user input. +# +# So the logic in this file applies some simple heuristics to guess the flags in +# those cases: +# - If the file is a header file (according to HEADER_EXTENSIONS) +# above, see if there is an entry in the database with the same root, but +# with any of the extensions in SOURCE_EXTENSIONS above. If so, return those +# flags. +# - Otherwise, if there is an entry in the database for the requested file, +# return those flags. +# - Otherwise, if we previously found any flags for a file in the same +# directory as the requested file, return those flags. +# - Otherwise, if we previously returned *any* flags for *any* file with the +# same file extension, return those. +# - Otherwise, if the file extension is known to us (c++, c, objective c), we +# return some absolutely arbitrary default flags, which might work for toy +# projects. +# - Otherwise, we really can't guess any flags, so return none (and thus a +# warning that we can't guess your flags). +# +# Caution +# ------- +# +# These heuristics might not work for you! Your mileage may vary. It is provided +# as a default in the hope that it might be useful for some number of users. +# +# The file is deliberately heavy on comments. This is to allow it to be adapted +# to individual use cases by copying it to a known directory, configuring it +# as the 'global_extra_conf' and customising it to your particular +# environment. Note that the comments in this file are not a replacement for the +# ycmd or YouCompleteMe documentation, nor is is meant to be a replacement for +# the comments/examples found in ycmd's own C++ source location +# (../cpp/ycm/.ycm_extra_conf.py). +# + +# Tweakables for heuristics {{{ + +# List of file extensions to be considered "header" files and thus not present +# in the compilation database. The logic will try and find an associated +# "source" file (see SOURCE_EXTENSIONS below) and use the flags for that. +HEADER_EXTENSIONS = [ '.h', '.hxx', '.hpp', '.hh' ] + +# List of file extensions which are considered "source" files for the purposes +# of heuristically locating the flags for a header file. +SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] + +# }}} + +# Caches for heuristically guessing flags {{{ + +# We cache the database for any given source directory +compilation_database_dir_map = {} + +# Sometimes we don't actually know what the flags to use are. Rather than +# returning no flags, if we've previously found flags for a file in a particular +# directory, return them. The will probably work in a high percentage of cases +# and allow new files (which are not yet in the compilation database) to receive +# at least some flags +file_directory_heuristic_map = {} + +# Assuming we can't find anything in the database, and we haven't previously +# seen any compilation information for the directory of the file in question, we +# simply return the last set of flags we successfully retrieved for any file +# sharing the file extension. This might work for some percentage of files, +# which is better than the 0% otherwise. +last_compilation_info_ext_map = {} + +# As a last gambit, we just return some generic flags. This won't work for +# any large or complex project, but it might work for trivial/toy projects, +# demos, school projects etc. + +# This map contains the flag lists to return for the supported file types. The +# flags are arbitrary, but should work for a large number of toy projects (when +# combined with the other os-specific flags added by ycmd). +fallback_flags_filetype_map = { + 'cpp': [ + '-x', 'c++', + '-std=c++11', + '-Wall', + '-Wextra', + '-I', '.' + ], + 'c': [ + '-x', 'c', + '-std=c99', + '-Wall', + '-Wextra', + '-I', '.' + ], + 'objc': [ + '-x', 'objective-c', + '-Wall', + '-Wextra', + '-I', '.' + ], +} + +# This map contains the mapping of file extension (including the .) to the file +# type we expect it to be. We would prefer to rely on the editor to tell us the +# file type, but we can't because there is no way to force that to happen. +fallback_flags_ext_map = { + '.cpp': fallback_flags_filetype_map[ 'cpp' ], + '.cxx': fallback_flags_filetype_map[ 'cpp' ], + '.cc': fallback_flags_filetype_map[ 'cpp' ], + '.c': fallback_flags_filetype_map[ 'c' ], + '.m': fallback_flags_filetype_map[ 'objc' ], + '.mm': fallback_flags_filetype_map[ 'objc' ], # sic: strictly obj-c++ ? +} + +# }}} + +# Implementation {{{ + + +# Return a compilation database object for the supplied path or None if none +# could be found. +# +# We search up the directory hierarchy, to first see if we have a compilation +# database already for that path, or if a compile_commands.json file exists in +# that directory. +def FindCompilationDatabase( wd ): + # Find all the ancestor directories of the supplied directory + for folder in utils.PathsToAllParentFolders( wd ): + # Did we already cache a database for this path? + if folder in compilation_database_dir_map: + # Yep. Return that. + return compilation_database_dir_map[ folder ] + + # Guess not. Let's see if a compile_commands.json file already exists... + compile_commands = os.path.join( folder, 'compile_commands.json' ) + if os.path.exists( compile_commands ): + # Yes, it exists. Create a database and cache it in our map. + database = ycm_core.CompilationDatabase( folder ) + compilation_database_dir_map[ folder ] = database + return database + + # Doesn't exist. Check the next ancestor + + # Nothing was found. No compilation flags are available. + # + # Note: we cache the fact that none was found for this folder to speed up + # subsequent searches.. + compilation_database_dir_map[ wd ] = None + return None + + +def HeaderPathManipulations( file_name ): + yield file_name + + # Try some simple manipulations, like removing "include" from the path and + # replacing with "src" or "lib" (as used by llvm) + + include_mappings = [ + [ '/include/', '/src/' ], + [ '/include/', '/lib/' ], + [ '/include/clang/', '/lib/' ], + ] + + for mapping in include_mappings: + if mapping[ 0 ] in file_name: + yield file_name.replace( mapping[ 0 ], mapping[ 1 ], 1 ) + + +# Find the compilation info structure from the supplied database for the +# supplied file. If the source file is a header, try and find an appropriate +# source file and return the compilation_info for that. +def GetCompilationInfoForFile( database, file_name, file_extension ): + + # The compilation_commands.json file generated by CMake does not have entries + # for header files. So we do our best by asking the db for flags for a + # corresponding source file, if any. If one exists, the flags for that file + # should be good enough. + if file_extension in HEADER_EXTENSIONS: + # It's a header file + for candidate_file in HeaderPathManipulations( file_name ): + for extension in SOURCE_EXTENSIONS: + replacement_file = os.path.splitext( candidate_file )[ 0 ] + extension + if os.path.exists( replacement_file ): + # We found a corresponding source file with the same file_root. Try + # and get the flags for that file. + compilation_info = database.GetCompilationInfoForFile( + replacement_file ) + if compilation_info.compiler_flags_: + return compilation_info + + # No corresponding source file was found, so we can't generate any flags for + # this header file. + return None + + # It's a source file. Just ask the database for the flags. + compilation_info = database.GetCompilationInfoForFile( file_name ) + if compilation_info.compiler_flags_: + return compilation_info + + return None + + +# In the absence of flags from a compilation database or heuristic, return some +# generic flags, or None if we don't have any generic flags for the supplied +# file extension. +def GetDefaultFlagsForFile( extension ): + if extension in fallback_flags_ext_map: + return { + 'flags': fallback_flags_ext_map[ extension ], + 'do_cache': True + } + else: + # OK we really have no clue about this filetype and we can't find any + # flags, even with outrageous guessing. Return nothing and warn the user + # that we couldn't find or guess any compiler flags. + return { + 'flags': [] + } + + +def MakeRelativePathsInFlagsAbsolute( flags, working_directory ): + if not working_directory: + return list( flags ) + new_flags = [] + make_next_absolute = False + path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ] + for flag in flags: + new_flag = flag + + if make_next_absolute: + make_next_absolute = False + if not flag.startswith( '/' ): + new_flag = os.path.join( working_directory, flag ) + + for path_flag in path_flags: + if flag == path_flag: + make_next_absolute = True + break + + if flag.startswith( path_flag ): + path = flag[ len( path_flag ): ] + new_flag = path_flag + os.path.join( working_directory, path ) + break + + if new_flag: + new_flags.append( new_flag ) + return new_flags + + +# ycmd calls this method to get the compile flags for a given file. It returns a +# dictionary with 2 keys: 'flags' and 'do_cache', or None if no flags can be +# found. +def FlagsForFile( file_name, **kwargs ): + file_dir = os.path.dirname( file_name ) + ( file_root, file_extension ) = os.path.splitext( file_name ) + + # Create or retrieve the cached compilation database object + database = FindCompilationDatabase( file_dir ) + if database is None: + # We couldn't find a compilation database. Just return some absolutely + # generic flags based on the file extension. + # + # We don't bother cacheing this as that would just be a waste of memory and + # cycles. + return GetDefaultFlagsForFile( file_extension ) + + compilation_info = GetCompilationInfoForFile( database, + file_name, + file_extension ) + + if compilation_info is None: + print( "No flags in database for " + file_name ) + if file_dir in file_directory_heuristic_map: + # We previously saw a file in this directory. As a guess, just + # return the flags for that file. Hopefully this will at least give some + # meaningful suggestions + print( " - Using flags for dir: " + file_dir ) + compilation_info = file_directory_heuristic_map[ file_dir ] + elif file_extension in last_compilation_info_ext_map: + # OK there is nothing in the DB for this file, and we didn't cache any + # flags for the directory of the file requested, but we did find some + # flags previously for this file type. Return them. Once again, this is + # an outrageous guess, but it is better to return *some* flags, rather + # than nothing. + print( " - Using flags for extension: " + file_extension ) + compilation_info = last_compilation_info_ext_map[ file_extension ] + else: + # No cache for this directory or file extension, we really can't conjure + # up any flags from the database, just return some absolutely generic + # fallback flags. These probably won't work for any actual project, but we + # try anyway. + print( " - Using generic flags" ) + return GetDefaultFlagsForFile( file_extension ) + + if file_dir not in file_directory_heuristic_map: + # This is the first file we've seen in path file_dir. Cache the + # compilation_info for it in case we see a file in the same dir with no + # flags available + file_directory_heuristic_map[ file_dir ] = compilation_info + + # Cache the last successful set of compilation info that we found. As noted + # above, this is used when we completely fail to find any flags. + last_compilation_info_ext_map[ file_extension ] = compilation_info + + return { + # We pass the compiler flags from the database unmodified. + 'flags': MakeRelativePathsInFlagsAbsolute( + compilation_info.compiler_flags_, + compilation_info.compiler_working_dir_ ), + + # We always want to use ycmd's cache, as this significantly improves + # performance. + 'do_cache': True + } + +# }}} diff --git a/ycmd/extra_conf_store.py b/ycmd/extra_conf_store.py index 533b6c9206..54e0ff46cd 100644 --- a/ycmd/extra_conf_store.py +++ b/ycmd/extra_conf_store.py @@ -42,6 +42,9 @@ _module_file_for_source_file = {} _module_file_for_source_file_lock = Lock() +YCMD_DEFAULT_EXTRA_CONF_PATH = os.path.join( os.path.dirname( __file__ ), + 'default_ycm_extra_conf.py' ) + def Get(): return _module_for_module_file, _module_file_for_source_file @@ -129,7 +132,9 @@ def _ShouldLoad( module_file, is_global ): decide using a white-/blacklist and ask the user for confirmation as a fallback.""" - if is_global or not user_options_store.Value( 'confirm_extra_conf' ): + if ( is_global or + not user_options_store.Value( 'confirm_extra_conf' ) or + module_file == YCMD_DEFAULT_EXTRA_CONF_PATH ): return True globlist = user_options_store.Value( 'extra_conf_globlist' ) @@ -202,7 +207,10 @@ def _MatchesGlobPattern( filename, glob ): def _ExtraConfModuleSourceFilesForFile( filename ): """For a given filename, search all parent folders for YCM_EXTRA_CONF_FILENAME files that will compute the flags necessary to compile the file. - If _GlobalYcmExtraConfFileLocation() exists it is returned as a fallback.""" + If _GlobalYcmExtraConfFileLocation() exists it is returned as a fallback. + Otherwise, we return the "default" extra_conf file. This file applies + best-efforts to generate flags according to a series of heuristics (such as + looking for a compilation database)""" for folder in PathsToAllParentFolders( filename ): candidate = os.path.join( folder, YCM_EXTRA_CONF_FILENAME ) @@ -213,6 +221,8 @@ def _ExtraConfModuleSourceFilesForFile( filename ): and os.path.exists( global_ycm_extra_conf ) ): yield global_ycm_extra_conf + yield YCMD_DEFAULT_EXTRA_CONF_PATH + def _PathToCppCompleterFolder(): """Returns the path to the 'cpp' completer folder. This is necessary From 2828cd83ed7e316979757f8d2b6c4caf15060300 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sun, 13 Oct 2019 17:25:00 +0100 Subject: [PATCH 2/9] vimspector: attach to process --- .vimspector.json | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.vimspector.json b/.vimspector.json index 300970e65b..90553173c3 100644 --- a/.vimspector.json +++ b/.vimspector.json @@ -47,6 +47,48 @@ "YCM_TEST_NO_RETRY": "1" } } + }, + "python - attach": { + "adapter": "vscode-python", + "variables": [ + { + "python": { + "shell": "/bin/bash -c 'if [ -z \"${dollar}VIRTUAL_ENV\" ]; then echo $$(which python); else echo \"${dollar}VIRTUAL_ENV/bin/python\"; fi'" + } + } + ], + "configuration": { + "name": "Python attach", + "type": "vscode-python", + "request": "attach", + + "pythonPath": "${python}", + "host": "localhost", + "port": "${port}" + } + }, + "python - attach ptvsd": { + "adapter": { + "name": "ptvsd", + "port": 1234 + }, + "variables": [ + { + "python": { + "shell": "/bin/bash -c 'if [ -z \"${dollar}VIRTUAL_ENV\" ]; then echo $$(which python); else echo \"${dollar}VIRTUAL_ENV/bin/python\"; fi'" + } + } + ], + "configuration": { + "name": "Python attach", + "type": "vscode-python", + "request": "attach", + "debugOptions": [ + "RedirectOutput", + "UnixClient", + "ShowReturnValue" + ] + } } } } From 007953af20390b93d5e6d893ca2b3d4a3ed904a1 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Mon, 4 Nov 2019 10:48:15 +0000 Subject: [PATCH 3/9] WIP: Ability to ignore extra conf files --- ycmd/completers/cpp/flags.py | 11 ++++- .../language_server_completer.py | 13 +++-- ycmd/extra_conf_support.py | 48 +++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 ycmd/extra_conf_support.py diff --git a/ycmd/completers/cpp/flags.py b/ycmd/completers/cpp/flags.py index 015ad537cc..0aa2ca450b 100644 --- a/ycmd/completers/cpp/flags.py +++ b/ycmd/completers/cpp/flags.py @@ -36,6 +36,7 @@ ToUnicode, CLANG_RESOURCE_DIR ) from ycmd.responses import NoExtraConfDetected +from ycmd.extra_conf_support import IgnoreExtraConf # -include-pch and --sysroot= must be listed before -include and --sysroot # respectively because the latter is a prefix of the former (and the algorithm @@ -173,7 +174,10 @@ def _GetFlagsFromExtraConfOrDatabase( self, filename, client_data ): # Load the flags from the extra conf file if one is found and is not global. module = extra_conf_store.ModuleForSourceFile( filename ) if module and not extra_conf_store.IsGlobalExtraConfModule( module ): - return _CallExtraConfFlagsForFile( module, filename, client_data ) + try: + return _CallExtraConfFlagsForFile( module, filename, client_data ) + except IgnoreExtraConf: + pass # Load the flags from the compilation database if any. database = self.LoadCompilationDatabase( filename ) @@ -182,7 +186,10 @@ def _GetFlagsFromExtraConfOrDatabase( self, filename, client_data ): # Load the flags from the global extra conf if set. if module: - return _CallExtraConfFlagsForFile( module, filename, client_data ) + try: + return _CallExtraConfFlagsForFile( module, filename, client_data ) + except IgnoreExtraConf: + pass # No compilation database and no extra conf found. Warn the user if not # already warned. diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index 596d36c0f3..9881cf3d10 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -37,6 +37,7 @@ from ycmd.completers.completer import Completer, CompletionsCache from ycmd.completers.completer_utils import GetFileContents, GetFileLines from ycmd.utils import LOGGER +from ycmd.extra_conf_support import IgnoreExtraConf from ycmd.completers.language_server import language_server_protocol as lsp @@ -1264,10 +1265,14 @@ def DefaultSettings( self, request_data ): def GetSettings( self, module, request_data ): if hasattr( module, 'Settings' ): - settings = module.Settings( - language = self.Language(), - filename = request_data[ 'filepath' ], - client_data = request_data[ 'extra_conf_data' ] ) + try: + settings = module.Settings( + language = self.Language(), + filename = request_data[ 'filepath' ], + client_data = request_data[ 'extra_conf_data' ] ) + except IgnoreExtraConf: + settings = None + if settings is not None: return settings diff --git a/ycmd/extra_conf_support.py b/ycmd/extra_conf_support.py new file mode 100644 index 0000000000..19477a2409 --- /dev/null +++ b/ycmd/extra_conf_support.py @@ -0,0 +1,48 @@ +# Copyright (C) 2011-2019 ycmd contributors +# +# This file is part of ycmd. +# +# ycmd is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ycmd is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ycmd. If not, see . + +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +# Not installing aliases from python-future; it's unreliable and slow. +from builtins import * # noqa + + +class IgnoreExtraConf( Exception ): + """Raise this exception from within a FlagsForFile or Settings function to + instruct ycmd to ignore this module for the current file. + + For example, if you wish to delegate to ycmd's built-in compilation database + support, you can write: + + from ycmd.extra_conf_support import IgnoreExtraConf + + def Settings( **kwargs ): + if kwargs[ 'language' ] == 'c-family': + # Use compilation database + raise IgnoreExtraConf() + + if kwargs[ 'language' ] == 'python': + ... + + This will then tell ycmd to use your compile_commands.json, or global extra + conf as if this local module doens't exist. + """ + pass + + From 2d812c24e15ab6658b98e0ab56014341db6933c6 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sat, 29 Dec 2018 21:27:15 +0000 Subject: [PATCH 4/9] Tolerate buggy servers sending Snippets --- .../language_server/language_server_completer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index 9881cf3d10..6a3de66063 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -2281,8 +2281,12 @@ def _InsertionTextForItem( request_data, item ): # We do not support completion types of "Snippet". This is implicit in that we # don't say it is a "capability" in the initialize request. # Abort this request if the server is buggy and ignores us. - assert lsp.INSERT_TEXT_FORMAT[ - item.get( 'insertTextFormat' ) or 1 ] == 'PlainText' + insert_type = lsp.INSERT_TEXT_FORMAT[ item.get( 'insertTextFormat' ) or 1 ] + + if insert_type != "PlainText": + raise IncompatibleCompletionException( 'Unsupported insertion type ' + 'supplied: {}'.format( + insert_type ) ) fixits = None From 64e121fba78719126f2937acb272bedf8e965ac8 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Mon, 31 Dec 2018 17:50:53 +0000 Subject: [PATCH 5/9] WIP: Support for multi-line completions (ish) --- .../language_server_completer.py | 104 +++++++++++------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index 6a3de66063..f8f84a0547 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -2278,18 +2278,6 @@ def _InsertionTextForItem( request_data, item ): selecting this completion - start_codepoint = the start column at which the text should be inserted )""" - # We do not support completion types of "Snippet". This is implicit in that we - # don't say it is a "capability" in the initialize request. - # Abort this request if the server is buggy and ignores us. - insert_type = lsp.INSERT_TEXT_FORMAT[ item.get( 'insertTextFormat' ) or 1 ] - - if insert_type != "PlainText": - raise IncompatibleCompletionException( 'Unsupported insertion type ' - 'supplied: {}'.format( - insert_type ) ) - - fixits = None - start_codepoint = request_data[ 'start_codepoint' ] # We will always have one of insertText or label if 'insertText' in item and item[ 'insertText' ]: @@ -2297,13 +2285,16 @@ def _InsertionTextForItem( request_data, item ): else: insertion_text = item[ 'label' ] - additional_text_edits = [] + fixits = [] + filepath = request_data[ 'filepath' ] + contents = None - # Per the protocol, textEdit takes precedence over insertText, and must be - # on the same line (and containing) the originally requested position. These - # are a pain, and require fixing up later in some cases, as most of our - # clients won't be able to apply arbitrary edits (only 'completion', as - # opposed to 'content assist'). + # Per the protocol, textEdit takes precedence over insertText, and the initial + # range of the edit must be on the same line (and containing) the originally + # requested position. + # + # These are a pain, and require fixing up later in some cases, and simply + # don't work in others. if 'textEdit' in item and item[ 'textEdit' ]: text_edit = item[ 'textEdit' ] start_codepoint = _GetCompletionItemStartCodepointOrReject( text_edit, @@ -2312,25 +2303,62 @@ def _InsertionTextForItem( request_data, item ): insertion_text = text_edit[ 'newText' ] if '\n' in insertion_text: - # jdt.ls can return completions which generate code, such as - # getters/setters and entire anonymous classes. + # servers can return completions which generate code, such as + # getters/setters and entire anonymous classes. These contain newlines in + # the generated textEdit. This is irksome because ycmd's clients don't + # necessarily support that. Certainly, Vim doesn't. # - # In order to support this we would need to do something like: - # - invent some insertion_text based on label/insertText (or perhaps - # '' - # - insert a textEdit in additionalTextEdits which deletes this - # insertion - # - or perhaps just modify this textEdit to undo that change? - # - or perhaps somehow support insertion_text of '' (this doesn't work - # because of filtering/sorting, etc.). - # - insert this textEdit in additionalTextEdits + # However, we do have 'fixits' in completion responses which we can lean + # on. + # + # In order to support this we: + # - use the item's label as the intial insertion text with the start + # codepoint set to the query codepoint + # - insert a textEdit in additionalTextEdits which deletes this + # insertion + # - insert another textEdit in additionalTextEdits which applies this + # textedit # # These textEdits would need a lot of fixing up and is currently out of # scope. # # These sorts of completions aren't really in the spirit of ycmd at the # moment anyway. So for now, we just ignore this candidate. - raise IncompatibleCompletionException( insertion_text ) + insertion_text = item[ 'label' ] + start_codepoint = request_data[ 'start_codepoint' ] + + # Add a fixit which removes the inserted label + completion_fixit_chunks = [ + responses.FixItChunk( + '', + responses.Range( + responses.Location( request_data[ 'line_num' ], + start_codepoint, + filepath ), + responses.Location( request_data[ 'line_num' ], + start_codepoint + len( insertion_text ), + filepath ), + ) + ) + ] + fixits.append( + responses.FixIt( completion_fixit_chunks[ 0 ].range.start_, + completion_fixit_chunks, + item[ 'label' ] ) + ) + # Add a fixit which applies this textEdit + contents = GetFileLines( request_data, filepath ) + completion_fixit_chunks = [ + responses.FixItChunk( + text_edit[ 'newText' ], + _BuildRange( contents, filepath, text_edit[ 'range' ] ) + ) + ] + fixits.append( + responses.FixIt( completion_fixit_chunks[ 0 ].range.start_, + completion_fixit_chunks, + item[ 'label' ] ) + ) else: # Calculate the start codepoint based on the overlapping text in the # insertion text and the existing text. This is the behavior of Visual @@ -2339,21 +2367,23 @@ def _InsertionTextForItem( request_data, item ): start_codepoint -= FindOverlapLength( request_data[ 'prefix' ], insertion_text ) - additional_text_edits.extend( item.get( 'additionalTextEdits' ) or [] ) - + additional_text_edits = item.get( 'additionalTextEdits' ) or [] if additional_text_edits: - filepath = request_data[ 'filepath' ] - contents = GetFileLines( request_data, filepath ) + + # We might have already extracted the contents + if contents is None: + contents = GetFileLines( request_data, filepath ) + chunks = [ responses.FixItChunk( e[ 'newText' ], _BuildRange( contents, filepath, e[ 'range' ] ) ) for e in additional_text_edits ] - fixits = responses.BuildFixItResponse( - [ responses.FixIt( chunks[ 0 ].range.start_, chunks ) ] ) + fixits.append( responses.FixIt( chunks[ 0 ].range.start_, chunks ) ) - return insertion_text, fixits, start_codepoint + extra_data = responses.BuildFixItResponse( fixits ) if fixits else None + return insertion_text, extra_data, start_codepoint def FindOverlapLength( line_value, insertion_text ): From 15435a002b2338d5b92d34d8a856ae17cac5ca6d Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Wed, 23 Jan 2019 22:35:49 +0000 Subject: [PATCH 6/9] WIP: For fixit-completions, include an indicator in the message to the clent. This allows the client to set the cursor position more sensibly, or at least in keeping with what vscode does --- cpp/ycm/ClangCompleter/FixIt.h | 2 ++ cpp/ycm/ycm_core.cpp | 3 ++- .../language_server_completer.py | 26 ++++++++++++------- ycmd/responses.py | 9 ++++--- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/cpp/ycm/ClangCompleter/FixIt.h b/cpp/ycm/ClangCompleter/FixIt.h index 4bf2f59fd3..4ceae17052 100644 --- a/cpp/ycm/ClangCompleter/FixIt.h +++ b/cpp/ycm/ClangCompleter/FixIt.h @@ -52,6 +52,8 @@ struct FixIt { /// multiple diagnostics offering different fixit options. The text is /// displayed to the user, allowing them choose which diagnostic to apply. std::string text; + + bool is_completion{ false }; }; } // namespace YouCompleteMe diff --git a/cpp/ycm/ycm_core.cpp b/cpp/ycm/ycm_core.cpp index 454f2cfe8d..dd118b920d 100644 --- a/cpp/ycm/ycm_core.cpp +++ b/cpp/ycm/ycm_core.cpp @@ -197,7 +197,8 @@ PYBIND11_MODULE( ycm_core, mod ) .def( py::init<>() ) .def_readonly( "chunks", &FixIt::chunks ) .def_readonly( "location", &FixIt::location ) - .def_readonly( "text", &FixIt::text ); + .def_readonly( "text", &FixIt::text ) + .def_readonly( "is_completion", &FixIt::is_completion ); py::bind_vector< std::vector< FixIt > >( mod, "FixItVector" ); diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index f8f84a0547..6b6eeebdd9 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -2292,9 +2292,6 @@ def _InsertionTextForItem( request_data, item ): # Per the protocol, textEdit takes precedence over insertText, and the initial # range of the edit must be on the same line (and containing) the originally # requested position. - # - # These are a pain, and require fixing up later in some cases, and simply - # don't work in others. if 'textEdit' in item and item[ 'textEdit' ]: text_edit = item[ 'textEdit' ] start_codepoint = _GetCompletionItemStartCodepointOrReject( text_edit, @@ -2303,6 +2300,12 @@ def _InsertionTextForItem( request_data, item ): insertion_text = text_edit[ 'newText' ] if '\n' in insertion_text: + # FIXME: If this logic actually worked, then we should do it for + # everything and not just hte multi-line completions ? Would that allow us + # to avoid the _GetCompletionItemStartCodepointOrReject logic ? Possibly, + # but it would look strange in the UI cycling through items. In any case, + # doing both should be complete. + # servers can return completions which generate code, such as # getters/setters and entire anonymous classes. These contain newlines in # the generated textEdit. This is irksome because ycmd's clients don't @@ -2318,16 +2321,16 @@ def _InsertionTextForItem( request_data, item ): # insertion # - insert another textEdit in additionalTextEdits which applies this # textedit - # - # These textEdits would need a lot of fixing up and is currently out of - # scope. - # - # These sorts of completions aren't really in the spirit of ycmd at the - # moment anyway. So for now, we just ignore this candidate. insertion_text = item[ 'label' ] start_codepoint = request_data[ 'start_codepoint' ] # Add a fixit which removes the inserted label + # + # TODO: Perhaps we should actually supply a completion with an empty + # insertion_text, than have the _client_ deal with this. One key advantage + # to that is that the client can then decide where to put the cursor. + # Currently, the Vim client puts the cursor in the wrong place (i.e. + # before the text, rather than after it). completion_fixit_chunks = [ responses.FixItChunk( '', @@ -2341,6 +2344,8 @@ def _InsertionTextForItem( request_data, item ): ) ) ] + # FIXME: The problem with this is that it _might_ break the offsets in any + # additionalTextEdits fixits.append( responses.FixIt( completion_fixit_chunks[ 0 ].range.start_, completion_fixit_chunks, @@ -2357,7 +2362,8 @@ def _InsertionTextForItem( request_data, item ): fixits.append( responses.FixIt( completion_fixit_chunks[ 0 ].range.start_, completion_fixit_chunks, - item[ 'label' ] ) + item[ 'label' ], + is_completion = True ) ) else: # Calculate the start codepoint based on the overlapping text in the diff --git a/ycmd/responses.py b/ycmd/responses.py index 52a7bdfc5a..a0c8bca987 100644 --- a/ycmd/responses.py +++ b/ycmd/responses.py @@ -202,11 +202,12 @@ class FixIt( object ): must be byte offsets into the UTF-8 encoded version of the appropriate buffer. """ - def __init__( self, location, chunks, text = '' ): + def __init__( self, location, chunks, text = '', is_completion = False ): """location of type Location, chunks of type list""" self.location = location self.chunks = chunks self.text = text + self.is_completion = is_completion class FixItChunk( object ): @@ -298,14 +299,16 @@ def BuildFixItData( fixit ): return { 'command': fixit.command, 'text': fixit.text, - 'resolve': fixit.resolve + 'resolve': fixit.resolve, + 'is_completion': fixit.is_completion } else: return { 'location': BuildLocationData( fixit.location ), 'chunks' : [ BuildFixitChunkData( x ) for x in fixit.chunks ], 'text': fixit.text, - 'resolve': False + 'resolve': False, + 'is_completion': fixit.is_completion, } return { From 5452893c88cc2ddcdc7618ce0029656f78edd138 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sun, 27 Jan 2019 09:07:21 +0000 Subject: [PATCH 7/9] WIP: Snippet support --- .../language_server_completer.py | 27 +++++++++++++++---- .../language_server_protocol.py | 1 + ycmd/responses.py | 5 +++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index 6b6eeebdd9..d6a2938302 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -1057,7 +1057,7 @@ def _CandidatesFromCompletionItems( self, items, resolve, request_data ): item[ '_resolved' ] = True try: - insertion_text, extra_data, start_codepoint = ( + insertion_text, snippet, extra_data, start_codepoint = ( _InsertionTextForItem( request_data, item ) ) except IncompatibleCompletionException: LOGGER.exception( 'Ignoring incompatible completion suggestion %s', @@ -1076,6 +1076,7 @@ def _CandidatesFromCompletionItems( self, items, resolve, request_data ): # we might modify insertion_text should we see a lower start codepoint. completions.append( _CompletionItemToCompletionData( insertion_text, + snippet, item, extra_data ) ) start_codepoints.append( start_codepoint ) @@ -2218,7 +2219,7 @@ def _DistanceOfPointToRange( point, range ): return 0 -def _CompletionItemToCompletionData( insertion_text, item, fixits ): +def _CompletionItemToCompletionData( insertion_text, snippet, item, fixits ): # Since we send completionItemKind capabilities, we guarantee to handle # values outside our value set and fall back to a default. try: @@ -2236,7 +2237,8 @@ def _CompletionItemToCompletionData( insertion_text, item, fixits ): detailed_info = item[ 'label' ] + '\n\n' + documentation, menu_text = item[ 'label' ], kind = kind, - extra_data = fixits ) + extra_data = fixits, + snippet = snippet ) def _FixUpCompletionPrefixes( completions, @@ -2274,13 +2276,21 @@ def _InsertionTextForItem( request_data, item ): Returns a tuple ( - insertion_text = the text to insert + - snippet = optional snippet text - fixits = ycmd fixit which needs to be applied additionally when selecting this completion - start_codepoint = the start column at which the text should be inserted )""" start_codepoint = request_data[ 'start_codepoint' ] + label = item[ 'label' ] + insertion_text_is_snippet = False # We will always have one of insertText or label if 'insertText' in item and item[ 'insertText' ]: + # 1 = PlainText + # 2 = Snippet + if lsp.INSERT_TEXT_FORMAT[ item.get( 'insertTextFormat', 1 ) ] == 'Snippet': + insertion_text_is_snippet = True + insertion_text = item[ 'insertText' ] else: insertion_text = item[ 'label' ] @@ -2299,7 +2309,7 @@ def _InsertionTextForItem( request_data, item ): insertion_text = text_edit[ 'newText' ] - if '\n' in insertion_text: + if '\n' in insertion_text and not insertion_text_is_snippet: # FIXME: If this logic actually worked, then we should do it for # everything and not just hte multi-line completions ? Would that allow us # to avoid the _GetCompletionItemStartCodepointOrReject logic ? Possibly, @@ -2321,6 +2331,9 @@ def _InsertionTextForItem( request_data, item ): # insertion # - insert another textEdit in additionalTextEdits which applies this # textedit + # + # On the other hand, if the insertion text is a snippet, then the snippet + # system will handle the expansion. insertion_text = item[ 'label' ] start_codepoint = request_data[ 'start_codepoint' ] @@ -2389,7 +2402,11 @@ def _InsertionTextForItem( request_data, item ): fixits.append( responses.FixIt( chunks[ 0 ].range.start_, chunks ) ) extra_data = responses.BuildFixItResponse( fixits ) if fixits else None - return insertion_text, extra_data, start_codepoint + + if insertion_text_is_snippet: + return label, insertion_text, extra_data, start_codepoint + + return insertion_text, None, extra_data, start_codepoint def FindOverlapLength( line_value, insertion_text ): diff --git a/ycmd/completers/language_server/language_server_protocol.py b/ycmd/completers/language_server/language_server_protocol.py index 601bef3e55..9fc9c16670 100644 --- a/ycmd/completers/language_server/language_server_protocol.py +++ b/ycmd/completers/language_server/language_server_protocol.py @@ -264,6 +264,7 @@ def Initialize( request_id, project_directory, settings ): 'valueSet': list( range( 1, len( ITEM_KIND ) ) ), }, 'completionItem': { + 'snippetSupport': True, 'documentationFormat': [ 'plaintext', 'markdown' diff --git a/ycmd/responses.py b/ycmd/responses.py index a0c8bca987..d367f496ee 100644 --- a/ycmd/responses.py +++ b/ycmd/responses.py @@ -116,7 +116,8 @@ def BuildCompletionData( insertion_text, detailed_info = None, menu_text = None, kind = None, - extra_data = None ): + extra_data = None, + snippet = None ): completion_data = { 'insertion_text': insertion_text } @@ -131,6 +132,8 @@ def BuildCompletionData( insertion_text, completion_data[ 'kind' ] = kind if extra_data: completion_data[ 'extra_data' ] = extra_data + if snippet: + completion_data[ 'snippet' ] = snippet return completion_data From 5289137c1e0f993d85940551de363361c403dc08 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Tue, 5 Nov 2019 19:04:24 +0000 Subject: [PATCH 8/9] WIP: Don't declare snippet support becaus snippets suck. broken servers, like json will still return them and we'll use our stuff --- .../language_server/language_server_completer.py | 7 ++++++- .../completers/language_server/language_server_protocol.py | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index d6a2938302..8f3e30e552 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -2337,6 +2337,11 @@ def _InsertionTextForItem( request_data, item ): insertion_text = item[ 'label' ] start_codepoint = request_data[ 'start_codepoint' ] + # FIXME: + # So forced-completion breaks here, as the textEdit is formulated to + # remove the existing "prefix". E.g. typing getT, the edit + # attempts to replace getT with the new code. + # Add a fixit which removes the inserted label # # TODO: Perhaps we should actually supply a completion with an empty @@ -2368,7 +2373,7 @@ def _InsertionTextForItem( request_data, item ): contents = GetFileLines( request_data, filepath ) completion_fixit_chunks = [ responses.FixItChunk( - text_edit[ 'newText' ], + text_edit[ 'newText' ], # FIXME: This could also be a Snippet _BuildRange( contents, filepath, text_edit[ 'range' ] ) ) ] diff --git a/ycmd/completers/language_server/language_server_protocol.py b/ycmd/completers/language_server/language_server_protocol.py index 9fc9c16670..601bef3e55 100644 --- a/ycmd/completers/language_server/language_server_protocol.py +++ b/ycmd/completers/language_server/language_server_protocol.py @@ -264,7 +264,6 @@ def Initialize( request_id, project_directory, settings ): 'valueSet': list( range( 1, len( ITEM_KIND ) ) ), }, 'completionItem': { - 'snippetSupport': True, 'documentationFormat': [ 'plaintext', 'markdown' From dc4b5d93eb204316b6ce8d2454eec7aa0a48f6ee Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Thu, 14 Nov 2019 18:09:06 +0000 Subject: [PATCH 9/9] WIP: Allow capabilities to be set to enable snippets only where needed --- .../language_server_completer.py | 17 +++++-- .../language_server_protocol.py | 17 +++++-- ycmd/utils.py | 50 ++++++++++++++++++- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index 8f3e30e552..8ccf37c73c 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -1283,12 +1283,18 @@ def GetSettings( self, module, request_data ): def _GetSettingsFromExtraConf( self, request_data ): - self._settings = self.DefaultSettings( request_data ) + ls = self.DefaultSettings( request_data ) + # In case no module is found + self._settings = { + 'ls': ls + } module = extra_conf_store.ModuleForSourceFile( request_data[ 'filepath' ] ) if module: - settings = self.GetSettings( module, request_data ) - self._settings.update( settings.get( 'ls', {} ) ) + self._settings = self.GetSettings( module, request_data ) + ls.update( self._settings.get( 'ls', {} ) ) + self._settings[ 'ls' ] = ls + # Only return the dir if it was found in the paths; we don't want to use # the path of the global extra conf as a project root dir. if not extra_conf_store.IsGlobalExtraConfModule( module ): @@ -1706,7 +1712,8 @@ def _SendInitialize( self, request_data, extra_conf_dir ): # clear how/where that is specified. msg = lsp.Initialize( request_id, self._project_directory, - self._settings ) + self._settings.get( 'ls', {} ), + self._settings.get( 'capabilities', {} ) ) def response_handler( response, message ): if message is None: @@ -1819,7 +1826,7 @@ def _HandleInitializeInPollThread( self, response ): # configuration should be send in response to a workspace/configuration # request? self.GetConnection().SendNotification( - lsp.DidChangeConfiguration( self._settings ) ) + lsp.DidChangeConfiguration( self._settings.get( 'ls', {} ) ) ) # Notify the other threads that we have completed the initialize exchange. self._initialize_response = None diff --git a/ycmd/completers/language_server/language_server_protocol.py b/ycmd/completers/language_server/language_server_protocol.py index 601bef3e55..8710f0b200 100644 --- a/ycmd/completers/language_server/language_server_protocol.py +++ b/ycmd/completers/language_server/language_server_protocol.py @@ -34,7 +34,8 @@ unquote, url2pathname, urlparse, - urljoin ) + urljoin, + UpdateDict ) Error = collections.namedtuple( 'RequestError', [ 'code', 'reason' ] ) @@ -233,7 +234,10 @@ def BuildResponse( request, parameters ): return _BuildMessageData( message ) -def Initialize( request_id, project_directory, settings ): +def Initialize( request_id, + project_directory, + settings, + capabilities ): """Build the Language Server initialize request""" return BuildRequest( request_id, 'initialize', { @@ -241,8 +245,11 @@ def Initialize( request_id, project_directory, settings ): 'rootPath': project_directory, 'rootUri': FilePathToUri( project_directory ), 'initializationOptions': settings, - 'capabilities': { - 'workspace': { 'applyEdit': True, 'documentChanges': True }, + 'capabilities': UpdateDict( { + 'workspace': { + 'applyEdit': True, + 'documentChanges': True, + }, 'textDocument': { 'codeAction': { 'codeActionLiteralSupport': { @@ -288,7 +295,7 @@ def Initialize( request_id, project_directory, settings ): }, }, }, - }, + }, capabilities ), } ) diff --git a/ycmd/utils.py b/ycmd/utils.py index fca525fbb1..90aed0564b 100644 --- a/ycmd/utils.py +++ b/ycmd/utils.py @@ -24,7 +24,12 @@ # Not installing aliases from python-future; it's unreliable and slow. from builtins import * # noqa -from future.utils import PY2, native +try: + import collections.abc as collections_abc +except ImportError: + import collections as collections_abc + +from future.utils import PY2, native, iteritems import copy import json import logging @@ -673,3 +678,46 @@ def GetClangResourceDir(): CLANG_RESOURCE_DIR = GetClangResourceDir() + + +def UpdateDict( target, override ): + """Apply the updates in |override| to the dict |target|. This is like + dict.update, but recursive. i.e. if the existing element is a dict, then + override elements of the sub-dict rather than wholesale replacing. + + e.g. + + UpdateDict( + { + 'outer': { 'inner': { 'key': 'oldValue', 'existingKey': True } } + }, + { + 'outer': { 'inner': { 'key': 'newValue' } }, + 'newKey': { 'newDict': True }, + } + ) + + yields: + + { + outer: { + inner: { + key: 'newValue', + exitingKey: True + } + }, + newKey: { newDict: True } + } + + """ + + for key, value in iteritems( override ): + current_value = target.get( key ) + if not isinstance( current_value, collections_abc.Mapping ): + target[ key ] = value + elif isinstance( value, collections_abc.Mapping ): + target[ key ] = UpdateDict( current_value, value ) + else: + target[ key ] = value + + return target