dolphin: Implement exapansion with reserved words (#620)

compiler: rename the Engine struct to Compiler
compiler: move methods onto the Compiler struct
compiler: Use backticks for reserved words in MySQL
codegen: Handle queries with backticks

Author: kyleconroy

internal/cmd/generate.go

@@ -208,8 +208,8 @@ func parseMySQL(e Env, name, dir string, sql config.SQL, combo config.CombinedSe
 }
 
 func parse(e Env, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
-	eng := compiler.NewEngine(sql, combo)
-	if err := eng.ParseCatalog(sql.Schema); err != nil {
+	c := compiler.NewCompiler(sql, combo)
+	if err := c.ParseCatalog(sql.Schema); err != nil {
 		fmt.Fprintf(stderr, "# package %s\n", name)
 		if parserErr, ok := err.(*multierr.Error); ok {
 			for _, fileErr := range parserErr.Errs() {
@@ -221,9 +221,9 @@ func parse(e Env, name, dir string, sql config.SQL, combo config.CombinedSetting
 		return nil, true
 	}
 	if parserOpts.Debug.DumpCatalog {
-		debug.Dump(eng.Catalog())
+		debug.Dump(c.Catalog())
 	}
-	if err := eng.ParseQueries(sql.Queries, parserOpts); err != nil {
+	if err := c.ParseQueries(sql.Queries, parserOpts); err != nil {
 		fmt.Fprintf(stderr, "# package %s\n", name)
 		if parserErr, ok := err.(*multierr.Error); ok {
 			for _, fileErr := range parserErr.Errs() {
@@ -234,5 +234,5 @@ func parse(e Env, name, dir string, sql config.SQL, combo config.CombinedSetting
 		}
 		return nil, true
 	}
-	return eng.Result(), false
+	return c.Result(), false
 }

internal/codegen/golang/gen.go

@@ -238,7 +238,7 @@ import (
 {{range .GoQueries}}
 {{if $.OutputQuery .SourceName}}
 const {{.ConstantName}} = {{$.Q}}-- name: {{.MethodName}} {{.Cmd}}
-{{.SQL}}
+{{escape .SQL}}
 {{$.Q}}
 
 {{if .Arg.EmitStruct}}
@@ -393,6 +393,7 @@ func generate(settings config.CombinedSettings, enums []Enum, structs []Struct,
 	funcMap := template.FuncMap{
 		"lowerTitle": codegen.LowerTitle,
 		"comment":    codegen.DoubleSlashComment,
+		"escape":     codegen.EscapeBacktick,
 		"imports":    i.Imports,
 	}
 

internal/codegen/utils.go

@@ -11,6 +11,22 @@ func LowerTitle(s string) string {
 	return string(a)
 }
 
+// Go string literals cannot contain backtick. If a string contains
+// a backtick, replace it the following way:
+//
+// input:
+// 	SELECT `group` FROM foo
+//
+// output:
+// 	SELECT ` + "`" + `group` + "`" + ` FROM foo
+//
+// The escaped string must be rendered inside an existing string literal
+//
+// A string cannot be escaped twice
+func EscapeBacktick(s string) string {
+	return strings.Replace(s, "`", "`+\"`\"+`", -1)
+}
+
 func DoubleSlashComment(s string) string {
 	return "// " + strings.ReplaceAll(s, "\n", "\n// ")
 }

internal/compiler/compile.go

@@ -83,11 +83,11 @@ func parseCatalog(p Parser, c *catalog.Catalog, schemas []string) error {
 	return nil
 }
 
-func parseQueries(p Parser, c *catalog.Catalog, queries []string, o opts.Parser) (*Result, error) {
+func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) {
 	var q []*Query
 	merr := multierr.New()
 	set := map[string]struct{}{}
-	files, err := sqlpath.Glob(queries)
+	files, err := sqlpath.Glob(c.conf.Queries)
 	if err != nil {
 		return nil, err
 	}
@@ -98,13 +98,13 @@ func parseQueries(p Parser, c *catalog.Catalog, queries []string, o opts.Parser)
 			continue
 		}
 		src := string(blob)
-		stmts, err := p.Parse(strings.NewReader(src))
+		stmts, err := c.parser.Parse(strings.NewReader(src))
 		if err != nil {
 			merr.Add(filename, src, 0, err)
 			continue
 		}
 		for _, stmt := range stmts {
-			query, err := parseQuery(p, c, stmt.Raw, src, o)
+			query, err := c.parseQuery(stmt.Raw, src, o)
 			if err == ErrUnsupportedStatementType {
 				continue
 			}
@@ -134,10 +134,10 @@ func parseQueries(p Parser, c *catalog.Catalog, queries []string, o opts.Parser)
 		return nil, merr
 	}
 	if len(q) == 0 {
-		return nil, fmt.Errorf("no queries contained in paths %s", strings.Join(queries, ","))
+		return nil, fmt.Errorf("no queries contained in paths %s", strings.Join(c.conf.Queries, ","))
 	}
 	return &Result{
-		Catalog: c,
+		Catalog: c.catalog,
 		Queries: q,
 	}, nil
 }

internal/compiler/engine.go

@@ -11,51 +11,49 @@ import (
 	"github.com/kyleconroy/sqlc/internal/sql/catalog"
 )
 
-// The Engine type only exists as a compatibility shim between the old dinosql
-// package and the new compiler package.
-type Engine struct {
+type Compiler struct {
 	conf    config.SQL
 	combo   config.CombinedSettings
 	catalog *catalog.Catalog
 	parser  Parser
 	result  *Result
 }
 
-func NewEngine(conf config.SQL, combo config.CombinedSettings) *Engine {
-	e := &Engine{conf: conf, combo: combo}
+func NewCompiler(conf config.SQL, combo config.CombinedSettings) *Compiler {
+	c := &Compiler{conf: conf, combo: combo}
 	switch conf.Engine {
 	case config.EngineXLemon:
-		e.parser = sqlite.NewParser()
-		e.catalog = catalog.New("main")
+		c.parser = sqlite.NewParser()
+		c.catalog = catalog.New("main")
 	case config.EngineMySQL, config.EngineXDolphin:
-		e.parser = dolphin.NewParser()
-		e.catalog = catalog.New("public") // TODO: What is the default database for MySQL?
+		c.parser = dolphin.NewParser()
+		c.catalog = catalog.New("public") // TODO: What is the default database for MySQL?
 	case config.EnginePostgreSQL:
-		e.parser = postgresql.NewParser()
-		e.catalog = postgresql.NewCatalog()
+		c.parser = postgresql.NewParser()
+		c.catalog = postgresql.NewCatalog()
 	default:
 		panic(fmt.Sprintf("unknown engine: %s", conf.Engine))
 	}
-	return e
+	return c
 }
 
-func (e *Engine) Catalog() *catalog.Catalog {
-	return e.catalog
+func (c *Compiler) Catalog() *catalog.Catalog {
+	return c.catalog
 }
 
-func (e *Engine) ParseCatalog(schema []string) error {
-	return parseCatalog(e.parser, e.catalog, schema)
+func (c *Compiler) ParseCatalog(schema []string) error {
+	return parseCatalog(c.parser, c.catalog, schema)
 }
 
-func (e *Engine) ParseQueries(queries []string, o opts.Parser) error {
-	r, err := parseQueries(e.parser, e.catalog, e.conf.Queries, o)
+func (c *Compiler) ParseQueries(queries []string, o opts.Parser) error {
+	r, err := c.parseQueries(o)
 	if err != nil {
 		return err
 	}
-	e.result = r
+	c.result = r
 	return nil
 }
 
-func (e *Engine) Result() *Result {
-	return e.result
+func (c *Compiler) Result() *Result {
+	return c.result
 }

internal/compiler/expand.go

@@ -4,14 +4,15 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/kyleconroy/sqlc/internal/config"
 	"github.com/kyleconroy/sqlc/internal/source"
 	"github.com/kyleconroy/sqlc/internal/sql/ast"
 	"github.com/kyleconroy/sqlc/internal/sql/ast/pg"
 	"github.com/kyleconroy/sqlc/internal/sql/astutils"
 	"github.com/kyleconroy/sqlc/internal/sql/lang"
 )
 
-func expand(qc *QueryCatalog, raw *ast.RawStmt) ([]source.Edit, error) {
+func (c *Compiler) expand(qc *QueryCatalog, raw *ast.RawStmt) ([]source.Edit, error) {
 	list := astutils.Search(raw, func(node ast.Node) bool {
 		switch node.(type) {
 		case *pg.DeleteStmt:
@@ -28,7 +29,7 @@ func expand(qc *QueryCatalog, raw *ast.RawStmt) ([]source.Edit, error) {
 	}
 	var edits []source.Edit
 	for _, item := range list.Items {
-		edit, err := expandStmt(qc, raw, item)
+		edit, err := c.expandStmt(qc, raw, item)
 		if err != nil {
 			return nil, err
 		}
@@ -37,14 +38,20 @@ func expand(qc *QueryCatalog, raw *ast.RawStmt) ([]source.Edit, error) {
 	return edits, nil
 }
 
-func quoteIdent(ident string) string {
+func (c *Compiler) quoteIdent(ident string) string {
+	// TODO: Add a method to the parser / engine for this instead
 	if lang.IsReservedKeyword(ident) {
-		return "\"" + ident + "\""
+		switch c.conf.Engine {
+		case config.EngineMySQL, config.EngineXDolphin:
+			return "`" + ident + "`"
+		default:
+			return "\"" + ident + "\""
+		}
 	}
 	return ident
 }
 
-func expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node) ([]source.Edit, error) {
+func (c *Compiler) expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node) ([]source.Edit, error) {
 	tables, err := sourceTables(qc, node)
 	if err != nil {
 		return nil, err
@@ -103,14 +110,14 @@ func expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node) ([]source.Edi
 			if scope != "" && scope != t.Rel.Name {
 				continue
 			}
-			tableName := quoteIdent(t.Rel.Name)
-			scopeName := quoteIdent(scope)
-			for _, c := range t.Columns {
-				cname := c.Name
+			tableName := c.quoteIdent(t.Rel.Name)
+			scopeName := c.quoteIdent(scope)
+			for _, column := range t.Columns {
+				cname := column.Name
 				if res.Name != nil {
 					cname = *res.Name
 				}
-				cname = quoteIdent(cname)
+				cname = c.quoteIdent(cname)
 				if scope != "" {
 					cname = scopeName + "." + cname
 				}
@@ -122,7 +129,7 @@ func expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node) ([]source.Edi
 		}
 		var old []string
 		for _, p := range parts {
-			old = append(old, quoteIdent(p))
+			old = append(old, c.quoteIdent(p))
 		}
 		edits = append(edits, source.Edit{
 			Location: res.Location - raw.StmtLocation,

internal/compiler/parse.go

@@ -13,7 +13,6 @@ import (
 	"github.com/kyleconroy/sqlc/internal/sql/ast"
 	"github.com/kyleconroy/sqlc/internal/sql/ast/pg"
 	"github.com/kyleconroy/sqlc/internal/sql/astutils"
-	"github.com/kyleconroy/sqlc/internal/sql/catalog"
 	"github.com/kyleconroy/sqlc/internal/sql/rewrite"
 	"github.com/kyleconroy/sqlc/internal/sql/validate"
 )
@@ -32,7 +31,7 @@ func rewriteNumberedParameters(refs []paramRef, raw *ast.RawStmt, sql string) ([
 	return edits, nil
 }
 
-func parseQuery(p Parser, c *catalog.Catalog, stmt ast.Node, src string, o opts.Parser) (*Query, error) {
+func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query, error) {
 	if o.Debug.DumpAST {
 		debug.Dump(stmt)
 	}
@@ -66,10 +65,10 @@ func parseQuery(p Parser, c *catalog.Catalog, stmt ast.Node, src string, o opts.
 	if rawSQL == "" {
 		return nil, errors.New("missing semicolon at end of file")
 	}
-	if err := validate.FuncCall(c, raw); err != nil {
+	if err := validate.FuncCall(c.catalog, raw); err != nil {
 		return nil, err
 	}
-	name, cmd, err := metadata.Parse(strings.TrimSpace(rawSQL), p.CommentSyntax())
+	name, cmd, err := metadata.Parse(strings.TrimSpace(rawSQL), c.parser.CommentSyntax())
 	if err != nil {
 		return nil, err
 	}
@@ -89,12 +88,12 @@ func parseQuery(p Parser, c *catalog.Catalog, stmt ast.Node, src string, o opts.
 		refs = uniqueParamRefs(refs)
 		sort.Slice(refs, func(i, j int) bool { return refs[i].ref.Number < refs[j].ref.Number })
 	}
-	params, err := resolveCatalogRefs(c, rvs, refs, namedParams)
+	params, err := resolveCatalogRefs(c.catalog, rvs, refs, namedParams)
 	if err != nil {
 		return nil, err
 	}
 
-	qc, err := buildQueryCatalog(c, raw.Stmt)
+	qc, err := buildQueryCatalog(c.catalog, raw.Stmt)
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +102,7 @@ func parseQuery(p Parser, c *catalog.Catalog, stmt ast.Node, src string, o opts.
 		return nil, err
 	}
 
-	expandEdits, err := expand(qc, raw)
+	expandEdits, err := c.expand(qc, raw)
 	if err != nil {
 		return nil, err
 	}
@@ -116,7 +115,7 @@ func parseQuery(p Parser, c *catalog.Catalog, stmt ast.Node, src string, o opts.
 
 	// If the query string was edited, make sure the syntax is valid
 	if expanded != rawSQL {
-		if _, err := p.Parse(strings.NewReader(expanded)); err != nil {
+		if _, err := c.parser.Parse(strings.NewReader(expanded)); err != nil {
 			return nil, fmt.Errorf("edited query syntax is invalid: %w", err)
 		}
 	}