diff --git a/ycmd/completers/completer.py b/ycmd/completers/completer.py index 474d346bad..d53dafc52a 100644 --- a/ycmd/completers/completer.py +++ b/ycmd/completers/completer.py @@ -370,6 +370,10 @@ def ComputeSignaturesInner( self, request_data ): return {} + def ComputeSemanticTokens( self, request_data ): + return {} + + def DefinedSubcommands( self ): subcommands = sorted( self.GetSubcommandsMap().keys() ) try: diff --git a/ycmd/completers/language_server/language_server_completer.py b/ycmd/completers/language_server/language_server_completer.py index 0ea319be6d..5780659084 100644 --- a/ycmd/completers/language_server/language_server_completer.py +++ b/ycmd/completers/language_server/language_server_completer.py @@ -137,7 +137,11 @@ class ResponseAbortedException( Exception ): class ResponseFailedException( Exception ): """Raised by LanguageServerConnection if a request returns an error""" - pass # pragma: no cover + def __init__( self, error ): + self.error_code = error.get( 'code' ) or 0 + self.error_message = error.get( 'message' ) or "No message" + super().__init__( f'Request failed: { self.error_code }: ' + f'{ self.error_message }' ) class IncompatibleCompletionException( Exception ): @@ -212,11 +216,7 @@ def AwaitResponse( self, timeout ): if 'error' in self._message: error = self._message[ 'error' ] - raise ResponseFailedException( - 'Request failed: ' - f'{ error.get( "code" ) or 0 }' - ': ' - f'{ error.get( "message" ) or "No message" }' ) + raise ResponseFailedException( error ) return self._message @@ -1279,7 +1279,7 @@ def ComputeCandidatesInner( self, request_data, codepoint ): if not self._is_completion_provider: return None, False - self._UpdateServerWithFileContents( request_data ) + self._UpdateServerWithCurrentFileContents( request_data ) request_id = self.GetConnection().NextRequestId() @@ -1503,6 +1503,7 @@ def SignatureHelpAvailable( self ): else: return responses.SignatureHelpAvailalability.NOT_AVAILABLE + def ComputeSignaturesInner( self, request_data ): if not self.ServerIsReady(): return {} @@ -1510,11 +1511,10 @@ def ComputeSignaturesInner( self, request_data ): if not self._server_capabilities.get( 'signatureHelpProvider' ): return {} - self._UpdateServerWithFileContents( request_data ) + self._UpdateServerWithCurrentFileContents( request_data ) request_id = self.GetConnection().NextRequestId() msg = lsp.SignatureHelp( request_id, request_data ) - response = self.GetConnection().GetResponse( request_id, msg, REQUEST_TIMEOUT_COMPLETION ) @@ -1543,6 +1543,63 @@ def ComputeSignaturesInner( self, request_data ): return result + def ComputeSemanticTokens( self, request_data ): + if not self._initialize_event.wait( REQUEST_TIMEOUT_COMPLETION ): + return {} + + if not self._ServerIsInitialized(): + return {} + + self._UpdateServerWithCurrentFileContents( request_data ) + + server_config = self._server_capabilities.get( 'semanticTokensProvider' ) + if server_config is None: + return {} + + atlas = TokenAtlas( server_config[ 'legend' ] ) + + server_full_support = server_config.get( 'full' ) + if server_full_support == {}: + server_full_support = True + + if not server_full_support: + return {} + + request_id = self.GetConnection().NextRequestId() + + # Retry up to 3 times to avoid ContentModified errors + MAX_RETRY = 3 + for i in range( MAX_RETRY ): + try: + response = self._connection.GetResponse( + request_id, + lsp.SemanticTokens( + request_id, + request_data ), + 3 * REQUEST_TIMEOUT_COMPLETION ) + break + except ResponseFailedException as e: + if i < ( MAX_RETRY - 1 ) and e.error_code == lsp.Errors.ContentModified: + continue + else: + raise + + if response is None: + return {} + + filename = request_data[ 'filepath' ] + contents = GetFileLines( request_data, filename ) + result = response.get( 'result' ) or {} + tokens = _DecodeSemanticTokens( atlas, + result.get( 'data' ) or [], + filename, + contents ) + + return { + 'tokens': tokens + } + + def GetDetailedDiagnostic( self, request_data ): self._UpdateServerWithFileContents( request_data ) @@ -1985,6 +2042,14 @@ def _AnySupportedFileType( self, file_types ): return False + def _UpdateServerWithCurrentFileContents( self, request_data ): + file_name = request_data[ 'filepath' ] + contents = GetFileContents( request_data, file_name ) + filetypes = request_data[ 'filetypes' ] + with self._server_info_mutex: + self._RefreshFileContentsUnderLock( file_name, contents, filetypes ) + + def _UpdateServerWithFileContents( self, request_data ): """Update the server with the current contents of all open buffers, and close any buffers no longer open. @@ -1997,6 +2062,32 @@ def _UpdateServerWithFileContents( self, request_data ): self._PurgeMissingFilesUnderLock( files_to_purge ) + def _RefreshFileContentsUnderLock( self, file_name, contents, file_types ): + file_state = self._server_file_state[ file_name ] + action = file_state.GetDirtyFileAction( contents ) + + LOGGER.debug( 'Refreshing file %s: State is %s/action %s', + file_name, + file_state.state, + action ) + + if action == lsp.ServerFileState.OPEN_FILE: + msg = lsp.DidOpenTextDocument( file_state, + file_types, + contents ) + + self.GetConnection().SendNotification( msg ) + elif action == lsp.ServerFileState.CHANGE_FILE: + # FIXME: DidChangeTextDocument doesn't actually do anything + # different from DidOpenTextDocument other than send the right + # message, because we don't actually have a mechanism for generating + # the diffs. This isn't strictly necessary, but might lead to + # performance problems. + msg = lsp.DidChangeTextDocument( file_state, contents ) + + self.GetConnection().SendNotification( msg ) + + def _UpdateDirtyFilesUnderLock( self, request_data ): for file_name, file_data in request_data[ 'file_data' ].items(): if not self._AnySupportedFileType( file_data[ 'filetypes' ] ): @@ -2007,29 +2098,10 @@ def _UpdateDirtyFilesUnderLock( self, request_data ): self.SupportedFiletypes() ) continue - file_state = self._server_file_state[ file_name ] - action = file_state.GetDirtyFileAction( file_data[ 'contents' ] ) + self._RefreshFileContentsUnderLock( file_name, + file_data[ 'contents' ], + file_data[ 'filetypes' ] ) - LOGGER.debug( 'Refreshing file %s: State is %s/action %s', - file_name, - file_state.state, - action ) - - if action == lsp.ServerFileState.OPEN_FILE: - msg = lsp.DidOpenTextDocument( file_state, - file_data[ 'filetypes' ], - file_data[ 'contents' ] ) - - self.GetConnection().SendNotification( msg ) - elif action == lsp.ServerFileState.CHANGE_FILE: - # FIXME: DidChangeTextDocument doesn't actually do anything - # different from DidOpenTextDocument other than send the right - # message, because we don't actually have a mechanism for generating - # the diffs. This isn't strictly necessary, but might lead to - # performance problems. - msg = lsp.DidChangeTextDocument( file_state, file_data[ 'contents' ] ) - - self.GetConnection().SendNotification( msg ) def _UpdateSavedFilesUnderLock( self, request_data ): @@ -3329,3 +3401,77 @@ def on_deleted( self, event ): with self._server._server_info_mutex: msg = lsp.DidChangeWatchedFiles( event.src_path, 'delete' ) self._server.GetConnection().SendNotification( msg ) + + +class TokenAtlas: + def __init__( self, legend ): + self.tokenTypes = legend[ 'tokenTypes' ] + self.tokenModifiers = legend[ 'tokenModifiers' ] + + +def _DecodeSemanticTokens( atlas, token_data, filename, contents ): + assert len( token_data ) % 5 == 0 + + class Token: + line = 0 + start_character = 0 + num_characters = 0 + token_type = 0 + token_modifiers = 0 + + def DecodeModifiers( self, tokenModifiers ): + modifiers = [] + bit_index = 0 + while True: + bit_value = pow( 2, bit_index ) + + if bit_value > self.token_modifiers: + break + + if self.token_modifiers & bit_value: + modifiers.append( tokenModifiers[ bit_index ] ) + + bit_index += 1 + + return modifiers + + + last_token = Token() + tokens = [] + + for token_index in range( 0, len( token_data ), 5 ): + token = Token() + + token.line = last_token.line + token_data[ token_index ] + + token.start_character = token_data[ token_index + 1 ] + if token.line == last_token.line: + token.start_character += last_token.start_character + + token.num_characters = token_data[ token_index + 2 ] + + token.token_type = token_data[ token_index + 3 ] + token.token_modifiers = token_data[ token_index + 4 ] + + tokens.append( { + 'range': responses.BuildRangeData( _BuildRange( + contents, + filename, + { + 'start': { + 'line': token.line, + 'character': token.start_character, + }, + 'end': { + 'line': token.line, + 'character': token.start_character + token.num_characters, + } + } + ) ), + 'type': atlas.tokenTypes[ token.token_type ], + 'modifiers': token.DecodeModifiers( atlas.tokenModifiers ) + } ) + + last_token = token + + return tokens diff --git a/ycmd/completers/language_server/language_server_protocol.py b/ycmd/completers/language_server/language_server_protocol.py index c7190f1a7b..3a062f1978 100644 --- a/ycmd/completers/language_server/language_server_protocol.py +++ b/ycmd/completers/language_server/language_server_protocol.py @@ -137,6 +137,33 @@ class Errors: 'TypeParameter', ] +TOKEN_TYPES = [ + 'namespace', + 'type', + 'class', + 'enum', + 'interface', + 'struct', + 'typeParameter', + 'parameter', + 'variable', + 'property', + 'enumMember', + 'event', + 'function', + 'method', + 'member', + 'macro', + 'keyword', + 'modifier', + 'comment', + 'string', + 'number', + 'regexp', + 'operator', +] + +TOKEN_MODIFIERS = [] class InvalidUriException( Exception ): """Raised when trying to convert a server URI to a file path but the scheme @@ -327,6 +354,17 @@ def Initialize( request_id, project_directory, extra_capabilities, settings ): 'markdown' ], }, + 'semanticTokens': { + 'requests': { + 'range': False, + 'full': { + 'delta': False + } + }, + 'tokenTypes': TOKEN_TYPES, + 'tokenModifiers': TOKEN_MODIFIERS, + 'tokenFormats': [ 'relative' ] + } }, 'synchronization': { 'didSave': True @@ -445,9 +483,7 @@ def DidCloseTextDocument( file_state ): def Completion( request_id, request_data, codepoint ): return BuildRequest( request_id, 'textDocument/completion', { - 'textDocument': { - 'uri': FilePathToUri( request_data[ 'filepath' ] ), - }, + 'textDocument': TextDocumentIdentifier( request_data ), 'position': Position( request_data[ 'line_num' ], request_data[ 'line_value' ], codepoint ), @@ -497,9 +533,7 @@ def Implementation( request_id, request_data ): def CodeAction( request_id, request_data, best_match_range, diagnostics ): return BuildRequest( request_id, 'textDocument/codeAction', { - 'textDocument': { - 'uri': FilePathToUri( request_data[ 'filepath' ] ), - }, + 'textDocument': TextDocumentIdentifier( request_data ), 'range': best_match_range, 'context': { 'diagnostics': diagnostics, @@ -509,9 +543,7 @@ def CodeAction( request_id, request_data, best_match_range, diagnostics ): def Rename( request_id, request_data, new_name ): return BuildRequest( request_id, 'textDocument/rename', { - 'textDocument': { - 'uri': FilePathToUri( request_data[ 'filepath' ] ), - }, + 'textDocument': TextDocumentIdentifier( request_data ), 'newName': new_name, 'position': Position( request_data[ 'line_num' ], request_data[ 'line_value' ], @@ -533,11 +565,15 @@ def DocumentSymbol( request_id, request_data ): } ) +def TextDocumentIdentifier( request_data ): + return { + 'uri': FilePathToUri( request_data[ 'filepath' ] ), + } + + def BuildTextDocumentPositionParams( request_data ): return { - 'textDocument': { - 'uri': FilePathToUri( request_data[ 'filepath' ] ), - }, + 'textDocument': TextDocumentIdentifier( request_data ), 'position': Position( request_data[ 'line_num' ], request_data[ 'line_value' ], request_data[ 'column_codepoint' ] ) @@ -560,18 +596,14 @@ def Position( line_num, line_value, column_codepoint ): def Formatting( request_id, request_data ): return BuildRequest( request_id, 'textDocument/formatting', { - 'textDocument': { - 'uri': FilePathToUri( request_data[ 'filepath' ] ), - }, + 'textDocument': TextDocumentIdentifier( request_data ), 'options': FormattingOptions( request_data ) } ) def RangeFormatting( request_id, request_data ): return BuildRequest( request_id, 'textDocument/rangeFormatting', { - 'textDocument': { - 'uri': FilePathToUri( request_data[ 'filepath' ] ), - }, + 'textDocument': TextDocumentIdentifier( request_data ), 'range': Range( request_data ), 'options': FormattingOptions( request_data ) } ) @@ -635,6 +667,18 @@ def ExecuteCommand( request_id, command, arguments ): } ) +def SemanticTokens( request_id, request_data, previous_result_id = None ): + if 'range' in request_data: + return BuildRequest( request_id, 'textDocument/semanticTokens/range', { + 'textDocument': TextDocumentIdentifier( request_data ), + 'range': Range( request_data ) + } ) + else: + return BuildRequest( request_id, 'textDocument/semanticTokens/full', { + 'textDocument': TextDocumentIdentifier( request_data ), + } ) + + def FilePathToUri( file_name ): return urljoin( 'file:', pathname2url( file_name ) ) diff --git a/ycmd/handlers.py b/ycmd/handlers.py index b7c43c32ad..3c60131e70 100644 --- a/ycmd/handlers.py +++ b/ycmd/handlers.py @@ -30,6 +30,7 @@ BuildResolveCompletionResponse, BuildSignatureHelpResponse, BuildSignatureHelpAvailableResponse, + BuildSemanticTokensResponse, SignatureHelpAvailalability, UnknownExtraConf ) from ycmd.request_wrap import RequestWrap @@ -178,6 +179,32 @@ def GetSignatureHelp(): BuildSignatureHelpResponse( signature_info, errors = errors ) ) +@app.post( '/semantic_tokens' ) +def GetSemanticTokens(): + LOGGER.info( 'Received semantic tokens request' ) + request_data = RequestWrap( request.json ) + + if not _server_state.FiletypeCompletionUsable( request_data[ 'filetypes' ], + silent = True ): + return _JsonResponse( BuildSemanticTokensResponse( None ) ) + + errors = None + semantic_tokens = None + + try: + filetype_completer = _server_state.GetFiletypeCompleter( + request_data[ 'filetypes' ] ) + semantic_tokens = filetype_completer.ComputeSemanticTokens( request_data ) + except Exception as exception: + LOGGER.exception( 'Exception from semantic completer during tokens request' ) + errors = [ BuildExceptionResponse( exception, traceback.format_exc() ) ] + + # No fallback for signature help. The general completer is unlikely to be able + # to offer anything of for that here. + return _JsonResponse( + BuildSemanticTokensResponse( semantic_tokens, errors = errors ) ) + + @app.post( '/filter_and_sort_candidates' ) def FilterAndSortCandidates(): # Not using RequestWrap because no need and the requests coming in aren't like diff --git a/ycmd/responses.py b/ycmd/responses.py index 2cf784e84a..70d7319dd7 100644 --- a/ycmd/responses.py +++ b/ycmd/responses.py @@ -152,6 +152,14 @@ def BuildSignatureHelpResponse( signature_info, errors = None ): } +def BuildSemanticTokensResponse( semantic_tokens, errors = None ): + return { + 'semantic_tokens': + semantic_tokens if semantic_tokens else {}, + 'errors': errors if errors else [], + } + + # location.column_number_ is a byte offset def BuildLocationData( location ): return {