未验证 提交 1088824f 编写于 作者: F Florian Verdonck 提交者: GitHub

Add code comments to trivia (#12854)

* Store the range of line comments in the untyped syntax tree.

* Store the range of block comments in the untyped syntax tree.
上级 edcbca40
......@@ -27,6 +27,7 @@ open FSharp.Compiler.Lexhelp
open FSharp.Compiler.NameResolution
open FSharp.Compiler.ParseHelpers
open FSharp.Compiler.Syntax
open FSharp.Compiler.SyntaxTrivia
open FSharp.Compiler.Syntax.PrettyNaming
open FSharp.Compiler.SyntaxTreeOps
open FSharp.Compiler.Text
......@@ -198,8 +199,10 @@ let PostParseModuleImpls (defaultNamespace, filename, isLastCompiland, ParsedImp
yield! GetScopedPragmasForHashDirective hd ]
let conditionalDirectives = LexbufIfdefStore.GetTrivia(lexbuf)
let codeComments = LexbufCommentStore.GetComments(lexbuf)
let trivia: ParsedImplFileInputTrivia = { ConditionalDirectives = conditionalDirectives; CodeComments = codeComments }
ParsedInput.ImplFile (ParsedImplFileInput (filename, isScript, qualName, scopedPragmas, hashDirectives, impls, isLastCompiland, { ConditionalDirectives = conditionalDirectives }))
ParsedInput.ImplFile (ParsedImplFileInput (filename, isScript, qualName, scopedPragmas, hashDirectives, impls, isLastCompiland, trivia))
let PostParseModuleSpecs (defaultNamespace, filename, isLastCompiland, ParsedSigFile (hashDirectives, specs), lexbuf: UnicodeLexing.Lexbuf) =
match specs |> List.rev |> List.tryPick (function ParsedSigFileFragment.NamedModule(SynModuleOrNamespaceSig(lid, _, _, _, _, _, _, _)) -> Some lid | _ -> None) with
......@@ -220,8 +223,10 @@ let PostParseModuleSpecs (defaultNamespace, filename, isLastCompiland, ParsedSig
yield! GetScopedPragmasForHashDirective hd ]
let conditionalDirectives = LexbufIfdefStore.GetTrivia(lexbuf)
let codeComments = LexbufCommentStore.GetComments(lexbuf)
let trivia: ParsedSigFileInputTrivia = { ConditionalDirectives = conditionalDirectives; CodeComments = codeComments }
ParsedInput.SigFile (ParsedSigFileInput (filename, qualName, scopedPragmas, hashDirectives, specs, { ConditionalDirectives = conditionalDirectives }))
ParsedInput.SigFile (ParsedSigFileInput (filename, qualName, scopedPragmas, hashDirectives, specs, trivia))
type ModuleNamesDict = Map<string,Map<string,QualifiedNameOfFile>>
......@@ -348,7 +353,7 @@ let EmptyParsedInput(filename, isLastCompiland) =
[],
[],
[],
{ ConditionalDirectives = [] }
{ ConditionalDirectives = []; CodeComments = [] }
)
)
else
......@@ -361,7 +366,7 @@ let EmptyParsedInput(filename, isLastCompiland) =
[],
[],
isLastCompiland,
{ ConditionalDirectives = [] }
{ ConditionalDirectives = []; CodeComments = [] }
)
)
......
......@@ -227,6 +227,37 @@ module LexbufIfdefStore =
let store = getStore lexbuf
Seq.toList store
/// Used to capture the ranges of code comments as syntax trivia
module LexbufCommentStore =
// The key into the BufferLocalStore used to hold the compiler directives
let private commentKey = "Comments"
let private getStore (lexbuf: Lexbuf): ResizeArray<CommentTrivia> =
match lexbuf.BufferLocalStore.TryGetValue commentKey with
| true, store -> store
| _ ->
let store = box (ResizeArray<CommentTrivia>())
lexbuf.BufferLocalStore.[commentKey] <- store
store
|> unbox<ResizeArray<CommentTrivia>>
let SaveSingleLineComment (lexbuf: Lexbuf, startRange: range, endRange: range) =
let store = getStore lexbuf
let m = unionRanges startRange endRange
store.Add(CommentTrivia.LineComment(m))
let SaveBlockComment (lexbuf: Lexbuf, startRange: range, endRange: range) =
let store = getStore lexbuf
let m = unionRanges startRange endRange
store.Add(CommentTrivia.BlockComment(m))
let GetComments (lexbuf: Lexbuf): CommentTrivia list =
let store = getStore lexbuf
Seq.toList store
let ClearComments (lexbuf: Lexbuf): unit =
lexbuf.BufferLocalStore.Remove(commentKey) |> ignore
//------------------------------------------------------------------------
// Parsing: continuations for whitespace tokens
//------------------------------------------------------------------------
......
......@@ -84,6 +84,16 @@ module LexbufIfdefStore =
val GetTrivia: lexbuf:UnicodeLexing.Lexbuf -> SyntaxTrivia.ConditionalDirectiveTrivia list
module LexbufCommentStore =
val SaveSingleLineComment: lexbuf:UnicodeLexing.Lexbuf * startRange: range * endRange: range -> unit
val SaveBlockComment: lexbuf:UnicodeLexing.Lexbuf * startRange: range * endRange: range -> unit
val GetComments: lexbuf:UnicodeLexing.Lexbuf -> SyntaxTrivia.CommentTrivia list
val ClearComments: lexbuf:UnicodeLexing.Lexbuf -> unit
[<RequireQualifiedAccess>]
type LexerStringStyle =
| Verbatim
......
......@@ -15,14 +15,21 @@ and [<RequireQualifiedAccess; NoEquality; NoComparison>] IfDirectiveExpression =
| Or of IfDirectiveExpression * IfDirectiveExpression
| Not of IfDirectiveExpression
| Ident of string
[<RequireQualifiedAccess; NoEquality; NoComparison>]
type CommentTrivia =
| LineComment of range: range
| BlockComment of range: range
[<NoEquality; NoComparison>]
type ParsedImplFileInputTrivia =
{ ConditionalDirectives: ConditionalDirectiveTrivia list }
{ ConditionalDirectives: ConditionalDirectiveTrivia list
CodeComments: CommentTrivia list }
[<NoEquality; NoComparison>]
type ParsedSigFileInputTrivia =
{ ConditionalDirectives: ConditionalDirectiveTrivia list }
{ ConditionalDirectives: ConditionalDirectiveTrivia list
CodeComments: CommentTrivia list }
[<NoEquality; NoComparison>]
type SynExprTryWithTrivia =
......
......@@ -16,12 +16,19 @@ type [<RequireQualifiedAccess; NoEquality; NoComparison>] IfDirectiveExpression
| Not of IfDirectiveExpression
| Ident of string
[<RequireQualifiedAccess; NoEquality; NoComparison>]
type CommentTrivia =
| LineComment of range: range
| BlockComment of range: range
/// Represents additional information for ParsedImplFileInput
[<NoEquality; NoComparison>]
type ParsedImplFileInputTrivia =
{
/// Preprocessor directives of type #if, #else or #endif
ConditionalDirectives: ConditionalDirectiveTrivia list
/// Represent code comments found in the source file
CodeComments: CommentTrivia list
}
/// Represents additional information for ParsedSigFileInputTrivia
......@@ -30,6 +37,8 @@ type ParsedSigFileInputTrivia =
{
/// Preprocessor directives of type #if, #else or #endif
ConditionalDirectives: ConditionalDirectiveTrivia list
/// Represent code comments found in the source file
CodeComments: CommentTrivia list
}
/// Represents additional information for SynExpr.TryWith
......
......@@ -1778,7 +1778,7 @@ type internal FsiDynamicCompiler
let impl = SynModuleOrNamespace(prefix,(*isRec*)false, SynModuleOrNamespaceKind.NamedModule,defs,PreXmlDoc.Empty,[],None,m)
let isLastCompiland = true
let isExe = false
let input = ParsedInput.ImplFile (ParsedImplFileInput (filename,true, ComputeQualifiedNameOfFileFromUniquePath (m,prefixPath),[],[],[impl],(isLastCompiland, isExe), { ConditionalDirectives = [] }))
let input = ParsedInput.ImplFile (ParsedImplFileInput (filename,true, ComputeQualifiedNameOfFileFromUniquePath (m,prefixPath),[],[],[impl],(isLastCompiland, isExe), { ConditionalDirectives = []; CodeComments = [] }))
let isIncrementalFragment = true
let istate,tcEnvAtEndOfLastInput,declaredImpls = ProcessInputs (ctok, errorLogger, istate, [input], showTypes, isIncrementalFragment, isInteractiveItExpr, prefix, m)
let tcState = istate.tcState
......
......@@ -679,7 +679,7 @@ rule token args skip = parse
let m = lexbuf.LexemeRange
LexbufLocalXmlDocStore.AddGrabPointDelayed(lexbuf)
if not skip then LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, 1, m))
else singleLineComment (None,1,m,args) skip lexbuf }
else singleLineComment (None,1,m,m,args) skip lexbuf }
| "///" op_char*
{ // Match exactly 3 slash, 4+ slash caught by preceding rule
......@@ -687,14 +687,14 @@ rule token args skip = parse
let doc = lexemeTrimLeft lexbuf 3
let sb = (new StringBuilder(100)).Append(doc)
if not skip then LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, 1, m))
else singleLineComment (Some (m, sb),1,m,args) skip lexbuf }
else singleLineComment (Some (m, sb),1,m,m,args) skip lexbuf }
| "//" op_char*
{ // Need to read all operator symbols too, otherwise it might be parsed by a rule below
let m = lexbuf.LexemeRange
LexbufLocalXmlDocStore.AddGrabPointDelayed(lexbuf)
if not skip then LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, 1, m))
else singleLineComment (None,1,m,args) skip lexbuf }
else singleLineComment (None,1,m,m,args) skip lexbuf }
| newline
{ newline lexbuf
......@@ -917,7 +917,7 @@ rule token args skip = parse
let m = lexbuf.LexemeRange
let tok = LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, 1, m))
let tok = shouldStartFile args lexbuf m (0,FSComp.SR.lexHashBangMustBeFirstInFile()) tok
if not skip then tok else singleLineComment (None,1,m,args) skip lexbuf }
if not skip then tok else singleLineComment (None,1,m,m,args) skip lexbuf }
| "#light" anywhite*
| ("#indent" | "#light") anywhite+ "\"on\""
......@@ -1421,28 +1421,30 @@ and tripleQuoteString sargs skip = parse
// Parsing single-line comment - we need to split it into words for Visual Studio IDE
and singleLineComment cargs skip = parse
| newline
{ let buff,_n, _m, args = cargs
{ let buff,_n, mStart, mEnd, args = cargs
trySaveXmlDoc lexbuf buff
newline lexbuf
// Saves the documentation (if we're collecting any) into a buffer-local variable.
if not skip then LINE_COMMENT (LexCont.Token(args.ifdefStack, args.stringNest))
else token args skip lexbuf }
else
LexbufCommentStore.SaveSingleLineComment(lexbuf, mStart, mEnd)
token args skip lexbuf }
| eof
{ let _, _n, _m, args = cargs
{ let _, _n, _mStart, _mEnd, args = cargs
// NOTE: it is legal to end a file with this comment, so we'll return EOF as a token
EOF (LexCont.Token(args.ifdefStack, args.stringNest)) }
| [^ ' ' '\n' '\r' ]+
| anywhite+
{ let buff, n, m, args = cargs
{ let buff, n, m, _, args = cargs
// Append the current token to the XML documentation if we're collecting it
tryAppendXmlDoc buff (lexeme lexbuf)
if not skip then LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, n, m))
else singleLineComment (buff, n, m, args) skip lexbuf }
else singleLineComment (buff, n, m, lexbuf.LexemeRange, args) skip lexbuf }
| surrogateChar surrogateChar
| _ { let _, _n, _m, args = cargs
| _ { let _, _n, _mStart, _mEnd, args = cargs
if not skip then LINE_COMMENT (LexCont.Token(args.ifdefStack, args.stringNest))
else token args skip lexbuf }
......@@ -1493,7 +1495,9 @@ and comment cargs skip = parse
else
LexbufLocalXmlDocStore.AddGrabPointDelayed(lexbuf)
if not skip then COMMENT (LexCont.Token(args.ifdefStack, args.stringNest))
else token args skip lexbuf }
else
LexbufCommentStore.SaveBlockComment(lexbuf, m, lexbuf.LexemeRange)
token args skip lexbuf }
| anywhite+
| [^ '\'' '(' '*' '\n' '\r' '"' ')' '@' ' ' '\t' ]+
......
......@@ -83,6 +83,8 @@ let mkLexargs (defines, lightStatus, resourceManager, ifdefStack, errorLogger, p
let reusingLexbufForParsing lexbuf f =
use unwindBuildPhase = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse
LexbufLocalXmlDocStore.ClearXmlDoc lexbuf
LexbufCommentStore.ClearComments lexbuf
try
f ()
with e ->
......
......@@ -121,7 +121,7 @@ module IncrementalBuildSyntaxTree =
[],
[],
isLastCompiland,
{ ConditionalDirectives = [] }
{ ConditionalDirectives = []; CodeComments = [] }
)
)
else
......
......@@ -722,7 +722,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf,
lexargs.ifdefStack <- ifdefs
lexargs.stringNest <- stringNest
// The first argument is 'None' because we don't need XML comments when called from VS tokenizer
Lexer.singleLineComment (None, n, m, lexargs) skip lexbuf
Lexer.singleLineComment (None, n, m, m, lexargs) skip lexbuf
| LexCont.StringInComment (ifdefs, stringNest, style, n, m) ->
lexargs.ifdefStack <- ifdefs
......
......@@ -9089,6 +9089,25 @@ FSharp.Compiler.Syntax.TyparStaticReq: Int32 GetHashCode(System.Collections.IEqu
FSharp.Compiler.Syntax.TyparStaticReq: Int32 Tag
FSharp.Compiler.Syntax.TyparStaticReq: Int32 get_Tag()
FSharp.Compiler.Syntax.TyparStaticReq: System.String ToString()
FSharp.Compiler.SyntaxTrivia.CommentTrivia
FSharp.Compiler.SyntaxTrivia.CommentTrivia+BlockComment: FSharp.Compiler.Text.Range get_range()
FSharp.Compiler.SyntaxTrivia.CommentTrivia+BlockComment: FSharp.Compiler.Text.Range range
FSharp.Compiler.SyntaxTrivia.CommentTrivia+LineComment: FSharp.Compiler.Text.Range get_range()
FSharp.Compiler.SyntaxTrivia.CommentTrivia+LineComment: FSharp.Compiler.Text.Range range
FSharp.Compiler.SyntaxTrivia.CommentTrivia+Tags: Int32 BlockComment
FSharp.Compiler.SyntaxTrivia.CommentTrivia+Tags: Int32 LineComment
FSharp.Compiler.SyntaxTrivia.CommentTrivia: Boolean IsBlockComment
FSharp.Compiler.SyntaxTrivia.CommentTrivia: Boolean IsLineComment
FSharp.Compiler.SyntaxTrivia.CommentTrivia: Boolean get_IsBlockComment()
FSharp.Compiler.SyntaxTrivia.CommentTrivia: Boolean get_IsLineComment()
FSharp.Compiler.SyntaxTrivia.CommentTrivia: FSharp.Compiler.SyntaxTrivia.CommentTrivia NewBlockComment(FSharp.Compiler.Text.Range)
FSharp.Compiler.SyntaxTrivia.CommentTrivia: FSharp.Compiler.SyntaxTrivia.CommentTrivia NewLineComment(FSharp.Compiler.Text.Range)
FSharp.Compiler.SyntaxTrivia.CommentTrivia: FSharp.Compiler.SyntaxTrivia.CommentTrivia+BlockComment
FSharp.Compiler.SyntaxTrivia.CommentTrivia: FSharp.Compiler.SyntaxTrivia.CommentTrivia+LineComment
FSharp.Compiler.SyntaxTrivia.CommentTrivia: FSharp.Compiler.SyntaxTrivia.CommentTrivia+Tags
FSharp.Compiler.SyntaxTrivia.CommentTrivia: Int32 Tag
FSharp.Compiler.SyntaxTrivia.CommentTrivia: Int32 get_Tag()
FSharp.Compiler.SyntaxTrivia.CommentTrivia: System.String ToString()
FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia
FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia+Else: FSharp.Compiler.Text.Range get_range()
FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia+Else: FSharp.Compiler.Text.Range range
......@@ -9155,15 +9174,19 @@ FSharp.Compiler.SyntaxTrivia.IfDirectiveExpression: Int32 Tag
FSharp.Compiler.SyntaxTrivia.IfDirectiveExpression: Int32 get_Tag()
FSharp.Compiler.SyntaxTrivia.IfDirectiveExpression: System.String ToString()
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.CommentTrivia] CodeComments
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.CommentTrivia] get_CodeComments()
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia] ConditionalDirectives
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia] get_ConditionalDirectives()
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: System.String ToString()
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia])
FSharp.Compiler.SyntaxTrivia.ParsedImplFileInputTrivia: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia], Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.CommentTrivia])
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.CommentTrivia] CodeComments
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.CommentTrivia] get_CodeComments()
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia] ConditionalDirectives
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia] get_ConditionalDirectives()
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: System.String ToString()
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia])
FSharp.Compiler.SyntaxTrivia.ParsedSigFileInputTrivia: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.ConditionalDirectiveTrivia], Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SyntaxTrivia.CommentTrivia])
FSharp.Compiler.SyntaxTrivia.SynBindingTrivia
FSharp.Compiler.SyntaxTrivia.SynBindingTrivia: FSharp.Compiler.SyntaxTrivia.SynBindingTrivia Zero
FSharp.Compiler.SyntaxTrivia.SynBindingTrivia: FSharp.Compiler.SyntaxTrivia.SynBindingTrivia get_Zero()
......
......@@ -3654,3 +3654,95 @@ let v : string = \"\"\"
| [] -> Assert.Pass()
| _ ->
Assert.Fail $"Unexpected trivia, got {trivia}"
module CodeComments =
let private getCommentTrivia isSignatureFile source =
let ast = (if isSignatureFile then getParseResultsOfSignatureFile else getParseResults) source
match ast with
| ParsedInput.ImplFile(ParsedImplFileInput(trivia = { CodeComments = trivia }))
| ParsedInput.SigFile(ParsedSigFileInput(trivia = { CodeComments = trivia })) -> trivia
[<Test>]
let ``comment on single line`` () =
let trivia =
getCommentTrivia false """
// comment!
foo()
"""
match trivia with
| [ CommentTrivia.LineComment mComment ] ->
assertRange (2, 0) (2, 11) mComment
| _ ->
Assert.Fail "Could not get valid AST"
[<Test>]
let ``comment on single line, signature file`` () =
let trivia =
getCommentTrivia true """
namespace Meh
// comment!
foo()
"""
match trivia with
| [ CommentTrivia.LineComment mComment ] ->
assertRange (3, 0) (3, 11) mComment
| _ ->
Assert.Fail "Could not get valid AST"
[<Test>]
let ``comment after source code`` () =
let trivia =
getCommentTrivia false """
foo() // comment!
"""
match trivia with
| [ CommentTrivia.LineComment mComment ] ->
assertRange (2, 6) (2, 17) mComment
| _ ->
Assert.Fail "Could not get valid AST"
[<Test>]
let ``comment after source code, signature file`` () =
let trivia =
getCommentTrivia true """
namespace Meh
val foo : int // comment!
"""
match trivia with
| [ CommentTrivia.LineComment mComment ] ->
assertRange (4, 14) (4, 25) mComment
| _ ->
Assert.Fail "Could not get valid AST"
[<Test>]
let ``block comment in source code`` () =
let trivia =
getCommentTrivia false """
let a (* b *) c = c + 42
"""
match trivia with
| [ CommentTrivia.BlockComment mComment ] ->
assertRange (2, 6) (2, 13) mComment
| _ ->
Assert.Fail "Could not get valid AST"
[<Test>]
let ``block comment in source code, signature file`` () =
let trivia =
getCommentTrivia true """
namespace Meh
val a (* b *) : int
"""
match trivia with
| [ CommentTrivia.BlockComment mComment ] ->
assertRange (4, 6) (4, 13) mComment
| _ ->
Assert.Fail "Could not get valid AST"
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册