format with semicolon and comments

ast/ast.go

@@ -517,6 +517,12 @@ func (t *SQLToken) String() string {
 	case *token.SQLWord:
 		return v.String()
 	case string:
+		if t.Kind == token.Comment {
+			return "--" + v
+		}
+		if t.Kind == token.MultilineComment {
+			return "/*" + v + "*/"
+		}
 		return v
 	default:
 		return " "
@@ -528,6 +534,12 @@ func (t *SQLToken) NoQuoteString() string {
 	case *token.SQLWord:
 		return v.NoQuoteString()
 	case string:
+		if t.Kind == token.Comment {
+			return "--" + v
+		}
+		if t.Kind == token.MultilineComment {
+			return "/*" + v + "*/"
+		}
 		return v
 	default:
 		return " "
@@ -539,6 +551,12 @@ func (t *SQLToken) Render(opts *RenderOptions) string {
 	case *token.SQLWord:
 		return renderSQLWord(v, opts)
 	case string:
+		if t.Kind == token.Comment {
+			return "--" + v
+		}
+		if t.Kind == token.MultilineComment {
+			return "/*" + v + "*/"
+		}
 		return v
 	default:
 		return " "

internal/formatter/formatter.go

@@ -235,6 +235,24 @@ func formatItem(node ast.Node, env *formatEnvironment) ast.Node {
 		results = append(results, linebreakNode)
 		results = append(results, env.genIndent()...)
 	}
+	commentAfterMatcher := astutil.NodeMatcher{
+		ExpectTokens: []token.Kind{
+			token.Comment,
+		},
+	}
+	if commentAfterMatcher.IsMatch(node) {
+		results = append(results, linebreakNode)
+	}
+
+	breakStatementAfterMatcher := astutil.NodeMatcher{
+		ExpectTokens: []token.Kind{
+			token.Semicolon,
+		},
+	}
+	if breakStatementAfterMatcher.IsMatch(node) {
+		results = append(results, linebreakNode)
+		env.indentLevelDown()
+	}
 
 	return &ast.ItemWith{Toks: results}
 }

parser/parser.go

@@ -79,9 +79,6 @@ func NewParser(src io.Reader, d dialect.Dialect) (*Parser, error) {
 
 	parsed := []ast.Node{}
 	for _, tok := range tokens {
-		if tok.Kind == token.Comment {
-			continue
-		}
 		parsed = append(parsed, ast.NewItem(tok))
 	}
 
@@ -529,6 +526,12 @@ func parseAliased(reader *astutil.NodeReader) ast.Node {
 	}
 }
 
+var commentInfixMatcher = astutil.NodeMatcher{
+	ExpectTokens: []token.Kind{
+		token.Comment,
+		token.MultilineComment,
+	},
+}
 var identifierListInfixMatcher = astutil.NodeMatcher{
 	ExpectTokens: []token.Kind{
 		token.Comma,
@@ -570,7 +573,7 @@ func parseIdentifierList(reader *astutil.NodeReader) ast.Node {
 		peekNode            ast.Node
 	)
 	for {
-		if !tmpReader.PeekNodeIs(true, identifierListTargetMatcher) {
+		if !tmpReader.PeekNodeIs(true, identifierListTargetMatcher) && !tmpReader.PeekNodeIs(true, commentInfixMatcher) {
 			// Include white space after the comma
 			peekIndex, peekNode := tmpReader.PeekNode(true)
 			if peekNode != nil {
@@ -584,6 +587,9 @@ func parseIdentifierList(reader *astutil.NodeReader) ast.Node {
 			}
 			break
 		}
+		for tmpReader.PeekNodeIs(true, commentInfixMatcher) {
+			tmpReader.NextNode(true)
+		}
 
 		peekIndex, peekNode = tmpReader.PeekNode(true)
 		idents = append(idents, peekNode)

parser/parser_test.go

@@ -84,42 +84,43 @@ func TestParseComments(t *testing.T) {
 			name:  "line comment with identiger",
 			input: "-- foo\nbar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 2, "\nbar")
+				testStatement(t, stmts[0], 3, "-- foo\nbar")
 
 				list := stmts[0].GetTokens()
-				testItem(t, list[0], "\n")
-				testIdentifier(t, list[1], "bar")
+				testItem(t, list[0], "-- foo")
+				testIdentifier(t, list[2], "bar")
 			},
 		},
 		{
 			name:  "range comment with identiger",
 			input: "/* foo */bar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 1, "bar")
+				testStatement(t, stmts[0], 2, "/* foo */bar")
 
 				list := stmts[0].GetTokens()
-				testIdentifier(t, list[0], "bar")
+				testIdentifier(t, list[1], "bar")
 			},
 		},
 		{
 			name:  "range comment with identiger list",
 			input: "foo, /* foo */bar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 1, "foo, bar")
+				testStatement(t, stmts[0], 1, "foo, /* foo */bar")
 
 				list := stmts[0].GetTokens()
-				testIdentifierList(t, list[0], "foo, bar")
+				testIdentifierList(t, list[0], "foo, /* foo */bar")
 			},
 		},
 		{
 			name:  "multi line range comment with identiger",
 			input: "/*\n * foo\n */\nbar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 2, "\nbar")
+				testStatement(t, stmts[0], 3, "/*\n   foo\n */\nbar")
 
 				list := stmts[0].GetTokens()
-				testItem(t, list[0], "\n")
-				testIdentifier(t, list[1], "bar")
+				testItem(t, list[0], "/*\n   foo\n */")
+				testItem(t, list[1], "\n")
+				testIdentifier(t, list[2], "bar")
 			},
 		},
 	}

token/kind.go

@@ -20,6 +20,8 @@ const (
 	Whitespace
 	// comment node
 	Comment
+	// multiline comment node
+	MultilineComment
 	// = operator
 	Eq
 	// != or <> operator

token/lexer.go

@@ -274,7 +274,7 @@ func (t *Tokenizer) next() (Kind, interface{}, error) {
 			if err != nil {
 				return ILLEGAL, str, err
 			}
-			return Comment, str, nil
+			return MultilineComment, str, nil
 		}
 		t.Col++
 		return Div, "/", nil

token/lexer_test.go

@@ -462,7 +462,7 @@ multiline
 comment */`,
 			out: []*Token{
 				{
-					Kind:  Comment,
+					Kind:  MultilineComment,
 					Value: " test\nmultiline\ncomment ",
 					From:  Pos{Line: 0, Col: 0},
 					To:    Pos{Line: 2, Col: 10},