From dafdc4e5eae8bce90eaa0b5bd31743a0471994d9 Mon Sep 17 00:00:00 2001 From: Chet Husk Date: Sun, 31 Dec 2023 15:04:03 -0600 Subject: [PATCH 1/2] initial work on nested language support --- src/Core/DTO.fs | 10 +++ src/Core/LanguageService.fs | 143 +++++++++++++++++++++++++------ src/Core/NestedLanguages.fs | 165 ++++++++++++++++++++++++++++++++++++ src/Core/Notifications.fs | 35 ++++++++ src/Ionide.FSharp.fsproj | 2 + src/fsharp.fs | 2 +- 6 files changed, 330 insertions(+), 27 deletions(-) create mode 100644 src/Core/NestedLanguages.fs create mode 100644 src/Core/Notifications.fs diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index 4be1a17e..b7fed007 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -3,6 +3,10 @@ [] module DTO = + module LSP = + type Position = { line: int; character: int } + type Range = { start: Position; ``end``: Position } + type Pos = { Line: int; Column: int } type ParseRequest = @@ -368,6 +372,12 @@ module DTO = { file: string tests: TestAdapterEntry[] } + type NestedLanguagesForFile = + { textDocument: {| uri: string; version: int |} + nestedLanguages: + {| language: string + ranges: LSP.Range[] |}[] } + type Result<'T> = { Kind: string; Data: 'T } type HelptextResult = Result diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index ab5d84a8..32ac3db7 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -14,28 +14,6 @@ open LanguageServer module node = Node.Api -module Notifications = - type DocumentParsedEvent = - { - uri: string - version: float - /// BEWARE: Live object, might have changed since the parsing - document: TextDocument - } - - let onDocumentParsedEmitter = vscode.EventEmitter.Create() - let onDocumentParsed = onDocumentParsedEmitter.event - - let private tooltipRequestedEmitter = vscode.EventEmitter.Create() - let tooltipRequested = tooltipRequestedEmitter.event - - let mutable notifyWorkspaceHandler - : Option -> unit> = - None - - let testDetectedEmitter = vscode.EventEmitter.Create() - let testDetected = testDetectedEmitter.event - module LanguageService = open Fable.Import.LanguageServer.Client @@ -603,8 +581,6 @@ Consider: workspace.createFileSystemWatcher (U2.Case1 "**/*.{fs,fsx}", true, true, false) let clientOpts = - let opts = createEmpty - let mutable initFails = 0 let initializationFailureHandler (error: U3) = @@ -648,8 +624,7 @@ Consider: synch.configurationSection <- Some !^ "FSharp" synch.fileEvents <- Some(!^ ResizeArray([ fileDeletedWatcher ])) - // this type needs to be updated on the bindings - DocumentSelector is a (string|DocumentFilter) [] now only. - // that's why we need to coerce it here. + let opts = createEmpty opts.documentSelector <- Some selector opts.synchronize <- Some synch opts.errorHandler <- Some errorHandling @@ -662,6 +637,108 @@ Consider: opts.initializationOptions <- Some !^(Some initOpts) opts?markdown <- createObj [ "isTrusted" ==> true; "supportHtml" ==> true ] + + let middleware = + Fable.Core.JsInterop.createObj + [ "provideHover", + box ( + System.Func> + (fun doc pos cTok next -> + logger.Info( + "Checking if position %s in document %s is a known virtual document", + pos, + doc + ) + + match NestedLanguages.tryGetVirtualDocumentInDocAtPosition (doc, pos) with + | None -> next $ (doc, pos, cTok) + | Some(nestedDocUri, nestedLanguage) -> + logger.Info( + "Found virtual document %s with language %s", + nestedDocUri.toString (true), + nestedLanguage + ) + + box ( + commands.executeCommand ( + "vscode.executeHoverProvider", + unbox nestedDocUri, + unbox pos + ) + ) + |> unbox) + ) + "provideDocumentHighlights", + box ( + System.Func> + (fun doc pos cTok next -> + logger.Info( + "Checking if position %s in document %s is a known virtual document", + pos, + doc + ) + + match NestedLanguages.tryGetVirtualDocumentInDocAtPosition (doc, pos) with + | None -> next $ (doc, pos, cTok) + | Some(nestedDocUri, nestedLanguage) -> + logger.Info( + "Found virtual document %s with language %s", + nestedDocUri, + nestedLanguage + ) + + box ( + commands.executeCommand ( + "vscode.executeDocumentHighlights", + unbox nestedDocUri, + unbox pos + ) + ) + |> unbox) + ) + "provideDocumentSemanticTokens", + box ( + System.Func>(fun doc cTok next -> + logger.Info("Checking if document %s has any known virtual documents", doc) + + match NestedLanguages.getAllVirtualDocsForDoc (doc) with + | [||] -> next $ (doc, cTok) + | nestedDocs -> + logger.Info("Found virtual documents %j", nestedDocs) + // call tokens for parent doc _plus_ all virtual docs + async { + let! (baseDocTokens: obj[]) = + Async.AwaitPromise(unbox (next $ (doc, cTok)): JS.Promise) + + logger.Info("Got tokens for base doc") + let allTokens = ResizeArray(baseDocTokens) + + for (nestedDocUri, language) in nestedDocs do + logger.Info("Getting tokens for %s", nestedDocUri.ToString()) + + let! tokens = + commands.executeCommand ( + "vscode.provideDocumentSemanticTokens", + [| unbox nestedDocUri |] + ) + |> unbox> + |> Async.AwaitPromise + + if not (isUndefined tokens) then + allTokens.AddRange(tokens) + + let allTokens = allTokens.ToArray() + logger.Info("Got tokens for nested docs: %j", [| allTokens |]) + + return allTokens + } + |> Async.StartAsPromise + |> box + |> unbox) + ) ] + + opts?middleware <- middleware + opts let cl = LanguageClient("FSharp", "F#", options, clientOpts, false) @@ -975,6 +1052,20 @@ Consider: cl.onNotification ("fsharp/testDetected", (fun (a: TestForFile) -> Notifications.testDetectedEmitter.fire a)) + Notifications.nestedLanguagesDetected.Invoke(fun languages -> + logger.Debug("Nested languages detected: %j", languages) + None) + |> ignore + + cl.onNotification ( + "fsharp/textDocument/nestedLanguages", + (fun (languages: NestedLanguagesForFile) -> + // create and manage virtual documents for the languages in the file given: + // * create synthetic document uri + // * add to map of synthetic documents with parent file/version key + Notifications.nestedLanguagesDetectedEmitter.fire languages) + ) + let start (c: ExtensionContext) = promise { try diff --git a/src/Core/NestedLanguages.fs b/src/Core/NestedLanguages.fs new file mode 100644 index 00000000..88cd4c74 --- /dev/null +++ b/src/Core/NestedLanguages.fs @@ -0,0 +1,165 @@ +namespace Ionide.VSCode.FSharp + +open Ionide.VSCode.FSharp.DTO +open Fable.Import.VSCode +open Fable.Import.VSCode.Vscode + +type VSCUri = Fable.Import.VSCode.Vscode.Uri + +[] +module NestedLanguages = + + type NestedDocument = + { languageId: string + content: string + rangesInParentFile: Range[] } + + type NestedDocuments = System.Collections.Generic.Dictionary + + let private documentsMap = NestedDocuments() + + let private documentsForFile = + System.Collections.Generic.Dictionary() + + let nestedDocumentScheme = "fsharp-nested-document" + + let private makeNestedDocumentName (parent: VSCUri) (languageId: string) (order: int) = + let uri = + let parentPathHash = parent.path.GetHashCode() + vscode.Uri.parse ($"{nestedDocumentScheme}:///{languageId}/{parentPathHash}/{order}.{languageId}", true) + + uri + + let private replaceWithWhitespace (input: string) (builder: System.Text.StringBuilder) = + input.Split([| '\n'; '\r' |], System.StringSplitOptions.None) + |> Array.iter (fun line -> + let s = System.String(' ', line.Length) + builder.Append(s) |> ignore) + + let private vscodePos (p: DTO.LSP.Position) : Vscode.Position = + vscode.Position.Create(p.line, p.character) + + let private vscodeRange (r: DTO.LSP.Range) : Vscode.Range = + vscode.Range.Create(vscodePos r.start, vscodePos r.``end``) + + let private createSyntheticDocument (parentDocument: TextDocument) (targetSubranges: DTO.LSP.Range[]) : string = + + let fullDocumentRange = + // create a range that covers the whole document by making a too-long range and having the document trim it to size + parentDocument.validateRange (vscode.Range.Create(0, 0, parentDocument.lineCount, 0)) + + let fullDocumentText = parentDocument.getText () + let builder = System.Text.StringBuilder(fullDocumentText.Length) + + match targetSubranges with + | [||] -> replaceWithWhitespace fullDocumentText builder + | [| single |] -> + replaceWithWhitespace + (parentDocument.getText (vscode.Range.Create(fullDocumentRange.start, vscodePos single.start))) + builder + + let actualContent = parentDocument.getText (vscodeRange single) + builder.Append(actualContent) |> ignore + + replaceWithWhitespace + (parentDocument.getText (vscode.Range.Create(vscodePos single.``end``, fullDocumentRange.``end``))) + builder + | ranges -> + let mutable currentPos = fullDocumentRange.start + // foreach range + // the space from currentPos to range.start is whitespace, copy that in + // the range is actual content, copy that in + // set the currentPos to range.end + // at the end of the ranges, copy in the whitespace from currentPos to fullDocumentRange.end + for range in ranges do + let currentToRangeStart = vscode.Range.Create(currentPos, vscodePos range.start) + replaceWithWhitespace (parentDocument.getText (currentToRangeStart)) builder + let actualContent = parentDocument.getText (vscodeRange range) + builder.Append(actualContent) |> ignore + currentPos <- vscodePos range.``end`` + + let currentToEnd = vscode.Range.Create(currentPos, fullDocumentRange.``end``) + replaceWithWhitespace (parentDocument.getText (currentToEnd)) builder + + builder.ToString() + + /// given the languages found in a given file, create a synthetic document for each language and track that + /// document in the documentsMap, clearing the documentsMap of any documents that are no longer needed for that file + let updateDocuments (languages: NestedLanguagesForFile) = + promise { + let parentDocumentUri = vscode.Uri.parse (languages.textDocument.uri, true) + let! parentDocument = workspace.openTextDocument (parentDocumentUri) + + // create virtual documents + let nestedDocuments = + languages.nestedLanguages + |> Array.mapi (fun order language -> + let uri = makeNestedDocumentName parentDocumentUri language.language order + let document = createSyntheticDocument parentDocument language.ranges + uri.toString (true), language.language, language.ranges |> Array.map vscodeRange, document) + + // track virtual documents with their parent + let uris = nestedDocuments |> Array.map (fun (fst, _, _, _) -> fst) + documentsForFile[languages.textDocument.uri] <- uris + + // TODO: remove documents when their parent closes + + // store the virtual contents in our map + nestedDocuments + |> Array.iter (fun (uri, language, ranges, document) -> + documentsMap[uri] <- + { languageId = language + rangesInParentFile = ranges + content = document }) + } + + let tryGetVirtualDocumentInDocAtPosition (parentDocument: TextDocument, position: Position) = + match documentsForFile.TryGetValue(parentDocument.uri.toString ()) with + | false, _ -> None + | true, nestedDocs -> + nestedDocs + |> Array.tryPick (fun nestedDocUri -> + match documentsMap.TryGetValue(nestedDocUri) with + | false, _ -> None + | true, nestedDoc -> + let containsPos = + nestedDoc.rangesInParentFile + |> Array.exists (fun range -> range.contains (Fable.Core.U2.Case1 position)) + + if containsPos then + Some(vscode.Uri.parse nestedDocUri, nestedDoc.languageId) + else + None) + + let getAllVirtualDocsForDoc (parentDocument: TextDocument) = + match documentsForFile.TryGetValue(parentDocument.uri.toString ()) with + | false, _ -> [||] + | true, nestedDocs -> + nestedDocs + |> Array.choose (fun nestedDocUri -> + match documentsMap.TryGetValue(nestedDocUri) with + | false, _ -> None + | true, nestedDoc -> Some(vscode.Uri.parse nestedDocUri, nestedDoc.languageId)) + + type private VirtualDocumentContentProvider() = + interface TextDocumentContentProvider with + member this.onDidChange: Event option = None + + member this.onDidChange + with set (v: Event option): unit = () + + member this.provideTextDocumentContent(uri: VSCUri, token: CancellationToken) : ProviderResult = + match documentsMap.TryGetValue(uri.toString (true)) with + | false, _ -> None + | true, nestedDoc -> unbox nestedDoc.content + + let activate (context: ExtensionContext) = + Notifications.nestedLanguagesDetected.Invoke(fun languages -> + updateDocuments languages |> ignore<_> + None) + |> context.Subscribe + + workspace.registerTextDocumentContentProvider (nestedDocumentScheme, VirtualDocumentContentProvider()) + |> context.Subscribe + + () diff --git a/src/Core/Notifications.fs b/src/Core/Notifications.fs new file mode 100644 index 00000000..62913f81 --- /dev/null +++ b/src/Core/Notifications.fs @@ -0,0 +1,35 @@ +namespace Ionide.VSCode.FSharp + +open Fable.Import +open Fable.Import.VSCode +open Fable.Import.VSCode.Vscode +open DTO + +[] +module Notifications = + type DocumentParsedEvent = + { + uri: string + version: float + /// BEWARE: Live object, might have changed since the parsing + document: TextDocument + } + + let onDocumentParsedEmitter = vscode.EventEmitter.Create() + let onDocumentParsed = onDocumentParsedEmitter.event + + let private tooltipRequestedEmitter = vscode.EventEmitter.Create() + let tooltipRequested = tooltipRequestedEmitter.event + + let mutable notifyWorkspaceHandler + : Option -> unit> = + None + + let testDetectedEmitter = vscode.EventEmitter.Create() + let testDetected = testDetectedEmitter.event + + let nestedLanguagesDetectedEmitter = + vscode.EventEmitter.Create() + + let nestedLanguagesDetected: Event = + nestedLanguagesDetectedEmitter.event diff --git a/src/Ionide.FSharp.fsproj b/src/Ionide.FSharp.fsproj index 6389a4f4..040cd00c 100644 --- a/src/Ionide.FSharp.fsproj +++ b/src/Ionide.FSharp.fsproj @@ -27,6 +27,8 @@ + + diff --git a/src/fsharp.fs b/src/fsharp.fs index 1d01d7e0..aefc1260 100644 --- a/src/fsharp.fs +++ b/src/fsharp.fs @@ -86,7 +86,7 @@ let activate (context: ExtensionContext) : JS.Promise = |> Promise.map (fun _ -> if solutionExplorer then tryActivate "solutionExplorer" SolutionExplorer.activate context - + tryActivate "nestedLanguages" NestedLanguages.activate context tryActivate "fsprojedit" FsProjEdit.activate context tryActivate "diagnostics" Diagnostics.activate context tryActivate "linelens" LineLens.Instance.activate context From 9b7205e4735d01d4fd6277b19b78d07fcb2c511b Mon Sep 17 00:00:00 2001 From: Chet Husk Date: Sat, 6 Jan 2024 15:50:13 -0600 Subject: [PATCH 2/2] tons of debugging and virtual document generation fixes --- src/Core/LanguageService.fs | 71 ++++++++++++++++++++++++---------- src/Core/NestedLanguages.fs | 77 ++++++++++++++++++++++++++++++------- 2 files changed, 113 insertions(+), 35 deletions(-) diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index 32ac3db7..9843b119 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -645,7 +645,7 @@ Consider: System.Func> (fun doc pos cTok next -> logger.Info( - "Checking if position %s in document %s is a known virtual document", + "[hover] Checking if position %s in document %s is a known virtual document", pos, doc ) @@ -654,52 +654,78 @@ Consider: | None -> next $ (doc, pos, cTok) | Some(nestedDocUri, nestedLanguage) -> logger.Info( - "Found virtual document %s with language %s", + "[hover] Found virtual document %s with language %s", nestedDocUri.toString (true), nestedLanguage ) - box ( - commands.executeCommand ( - "vscode.executeHoverProvider", - unbox nestedDocUri, - unbox pos + promise { + let! doc = workspace.openTextDocument (nestedDocUri) |> Promise.ofThenable + let charRange = vscode.Range.Create(pos, pos.translate (characterDelta = 1)) + + let content = + doc + .getText(charRange) + .Split([| Environment.NewLine |], StringSplitOptions.None) + + logger.Info( + "[hover] Virtual document %s has content '%j' at position %s", + nestedDocUri.toString (true), + content, + pos ) - ) - |> unbox) + + let! results = + commands.executeCommand ( + "vscode.executeHoverProvider", + unbox nestedDocUri, + unbox pos + ) + |> Promise.ofThenable + |> Promise.catch (fun e -> + logger.Error("Error while executing hover provider: %o", e) + None) + + return results + } + |> Promise.toThenable + |> U2.Case2 + |> Some) ) "provideDocumentHighlights", box ( System.Func> (fun doc pos cTok next -> logger.Info( - "Checking if position %s in document %s is a known virtual document", + "[highlights] Checking if position %s in document %s is a known virtual document", pos, - doc + doc.uri.toString (true) ) match NestedLanguages.tryGetVirtualDocumentInDocAtPosition (doc, pos) with | None -> next $ (doc, pos, cTok) | Some(nestedDocUri, nestedLanguage) -> logger.Info( - "Found virtual document %s with language %s", - nestedDocUri, + "[highlights] Found virtual document %s with language %s", + nestedDocUri.toString (true), nestedLanguage ) - box ( - commands.executeCommand ( - "vscode.executeDocumentHighlights", - unbox nestedDocUri, - unbox pos - ) + + commands.executeCommand ( + "vscode.executeDocumentHighlights", + unbox nestedDocUri, + unbox pos ) + |> Promise.ofThenable + |> Promise.catchEnd (fun e -> + logger.Error("Error while executing highlights: %o", e)) |> unbox) ) "provideDocumentSemanticTokens", box ( System.Func>(fun doc cTok next -> - logger.Info("Checking if document %s has any known virtual documents", doc) + logger.Info("Checking if document %s has any known virtual documents", doc.uri) match NestedLanguages.getAllVirtualDocsForDoc (doc) with | [||] -> next $ (doc, cTok) @@ -721,7 +747,10 @@ Consider: "vscode.provideDocumentSemanticTokens", [| unbox nestedDocUri |] ) - |> unbox> + |> Promise.ofThenable + |> Promise.catchEnd (fun e -> + logger.Error("Error while executing getting tokens: %o", e)) + |> unbox |> Async.AwaitPromise if not (isUndefined tokens) then diff --git a/src/Core/NestedLanguages.fs b/src/Core/NestedLanguages.fs index 88cd4c74..36355aea 100644 --- a/src/Core/NestedLanguages.fs +++ b/src/Core/NestedLanguages.fs @@ -9,6 +9,9 @@ type VSCUri = Fable.Import.VSCode.Vscode.Uri [] module NestedLanguages = + let private logger = + ConsoleAndOutputChannelLogger(Some "NestedLanguages", Level.DEBUG, Some defaultOutputChannel, Some Level.DEBUG) + type NestedDocument = { languageId: string content: string @@ -30,11 +33,10 @@ module NestedLanguages = uri - let private replaceWithWhitespace (input: string) (builder: System.Text.StringBuilder) = - input.Split([| '\n'; '\r' |], System.StringSplitOptions.None) - |> Array.iter (fun line -> - let s = System.String(' ', line.Length) - builder.Append(s) |> ignore) + let split (s: string) = + s.Split([| Node.Api.os.EOL |], System.StringSplitOptions.None) + + let empty len = System.String(Array.replicate len ' ') let private vscodePos (p: DTO.LSP.Position) : Vscode.Position = vscode.Position.Create(p.line, p.character) @@ -42,28 +44,40 @@ module NestedLanguages = let private vscodeRange (r: DTO.LSP.Range) : Vscode.Range = vscode.Range.Create(vscodePos r.start, vscodePos r.``end``) + let convertToWhitespace (s: string) (builder: System.Text.StringBuilder) = + split s + |> Array.iteri (fun index line -> + if index <> 0 then + builder.Append(Node.Api.os.EOL) |> ignore + + builder.Append(empty line.Length) |> ignore) + let private createSyntheticDocument (parentDocument: TextDocument) (targetSubranges: DTO.LSP.Range[]) : string = let fullDocumentRange = // create a range that covers the whole document by making a too-long range and having the document trim it to size parentDocument.validateRange (vscode.Range.Create(0, 0, parentDocument.lineCount, 0)) + logger.Info("Document %s has range %s", parentDocument.uri, fullDocumentRange) + let fullDocumentText = parentDocument.getText () let builder = System.Text.StringBuilder(fullDocumentText.Length) match targetSubranges with - | [||] -> replaceWithWhitespace fullDocumentText builder + | [||] -> convertToWhitespace fullDocumentText builder + | [| single |] -> - replaceWithWhitespace + convertToWhitespace (parentDocument.getText (vscode.Range.Create(fullDocumentRange.start, vscodePos single.start))) builder let actualContent = parentDocument.getText (vscodeRange single) builder.Append(actualContent) |> ignore - replaceWithWhitespace + convertToWhitespace (parentDocument.getText (vscode.Range.Create(vscodePos single.``end``, fullDocumentRange.``end``))) builder + | ranges -> let mutable currentPos = fullDocumentRange.start // foreach range @@ -73,15 +87,31 @@ module NestedLanguages = // at the end of the ranges, copy in the whitespace from currentPos to fullDocumentRange.end for range in ranges do let currentToRangeStart = vscode.Range.Create(currentPos, vscodePos range.start) - replaceWithWhitespace (parentDocument.getText (currentToRangeStart)) builder + + convertToWhitespace (parentDocument.getText (currentToRangeStart)) builder + let actualContent = parentDocument.getText (vscodeRange range) builder.Append(actualContent) |> ignore currentPos <- vscodePos range.``end`` let currentToEnd = vscode.Range.Create(currentPos, fullDocumentRange.``end``) - replaceWithWhitespace (parentDocument.getText (currentToEnd)) builder - builder.ToString() + convertToWhitespace (parentDocument.getText (currentToEnd)) builder + + let finalContent = builder.ToString() + + let finalLines = + finalContent.Split([| Node.Api.os.EOL |], System.StringSplitOptions.None) + + if float finalLines.Length <> parentDocument.lineCount then + logger.Error( + "Document %s has %d lines but synthetic document has %d lines", + parentDocument.uri.toString (), + parentDocument.lineCount, + finalLines.Length + ) + + finalContent /// given the languages found in a given file, create a synthetic document for each language and track that /// document in the documentsMap, clearing the documentsMap of any documents that are no longer needed for that file @@ -101,9 +131,6 @@ module NestedLanguages = // track virtual documents with their parent let uris = nestedDocuments |> Array.map (fun (fst, _, _, _) -> fst) documentsForFile[languages.textDocument.uri] <- uris - - // TODO: remove documents when their parent closes - // store the virtual contents in our map nestedDocuments |> Array.iter (fun (uri, language, ranges, document) -> @@ -111,6 +138,28 @@ module NestedLanguages = { languageId = language rangesInParentFile = ranges content = document }) + + // TODO: remove documents when their parent closes + let! _ = + nestedDocuments + |> Array.map (fun (uriString, lang, _, _) -> + promise { + let uri = vscode.Uri.parse (uriString, strict = true) + let! doc = workspace.openTextDocument (uri) + let actualRange = doc.validateRange (vscode.Range.Create(0, 0, 10000000, 10000000)) + + logger.Info( + "Classified document %s as language %s with content %s occupying range %s", + uriString, + doc.languageId, + doc.getText (), + actualRange + ) + }) + |> unbox + |> Promise.all + + return () } let tryGetVirtualDocumentInDocAtPosition (parentDocument: TextDocument, position: Position) =