blob: 75fa6c7fe7002076f6af774d7b621e42da09567c [file] [log] [blame]
// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package parse implements the VDL parser, converting source files into a parse
// tree. The ParseFile function is the main entry point.
package parse
//go:generate ./grammar_gen.sh
// This is the only file in this package that uses the yacc-generated parser
// with entrypoint yyParse. The result of the parse is the simple parse.File
// representation, which is used by the compilation stage.
//
// TODO(toddw): The yacc-generated parser returns pretty lousy error messages;
// basically "syntax error" is the only string returned. Improve them.
import (
"fmt"
"io"
"log"
"math/big"
"path"
"strconv"
"strings"
"text/scanner"
"v.io/x/ref/lib/vdl/vdlutil"
)
// Opts specifies vdl parsing options.
type Opts struct {
ImportsOnly bool // Only parse imports; skip everything else.
}
// ParseFile takes a file name, the contents of the vdl file src, and the
// accumulated errors, and parses the vdl into a parse.File containing the parse
// tree. Returns nil if any errors are encountered, with errs containing more
// information. Otherwise returns the parsed File.
func ParseFile(fileName string, src io.Reader, opts Opts, errs *vdlutil.Errors) *File {
start := startFile
if opts.ImportsOnly {
start = startFileImports
}
return parse(fileName, src, start, errs)
}
// ParseConfig takes a file name, the contents of the config file src, and the
// accumulated errors, and parses the config into a parse.Config containing the
// parse tree. Returns nil if any errors are encountered, with errs containing
// more information. Otherwise returns the parsed Config.
func ParseConfig(fileName string, src io.Reader, opts Opts, errs *vdlutil.Errors) *Config {
start := startConfig
if opts.ImportsOnly {
start = startConfigImports
}
// Since the syntax is so similar between config files and vdl files, we just
// parse it as a vdl file and populate Config afterwards.
file := parse(fileName, src, start, errs)
if file == nil {
return nil
}
if len(file.ErrorDefs) > 0 || len(file.TypeDefs) > 0 || len(file.Interfaces) > 0 {
errs.Errorf("%s: config files may not contain error, type or interface definitions", fileName)
return nil
}
config := &Config{
FileName: fileName,
Doc: file.Doc,
ConfigDef: file.PackageDef,
Imports: file.Imports,
Config: file.ConstDefs[0].Expr,
ConstDefs: file.ConstDefs[1:],
}
if len(config.ConstDefs) == 0 {
config.ConstDefs = nil
}
if opts.ImportsOnly {
// Clear out the const expression from the config clause.
config.Config = nil
config.ConstDefs = nil
}
return config
}
func parse(fileName string, src io.Reader, startTok int, errs *vdlutil.Errors) *File {
if errs == nil {
log.Fatal("Nil errors specified for Parse")
}
origErrs := errs.NumErrors()
lex := newLexer(fileName, src, startTok, errs)
if errCode := yyParse(lex); errCode != 0 {
errs.Errorf("%s: yyParse returned error code %v", fileName, errCode)
}
lex.attachComments()
if startTok == startFile || startTok == startConfig {
vdlutil.Vlog.Printf("PARSE RESULTS\n\n%v\n\n", lex.vdlFile)
}
if origErrs != errs.NumErrors() {
return nil
}
return lex.vdlFile
}
// ParseExprs parses data into a slice of parsed const expressions. The input
// data is specified in VDL syntax, with commas separating multiple expressions.
// There must be at least one expression specified in data. Errors are returned
// in errs.
func ParseExprs(data string, errs *vdlutil.Errors) []ConstExpr {
const name = "exprs"
lex := newLexer(name, strings.NewReader(data), startExprs, errs)
if errCode := yyParse(lex); errCode != 0 {
errs.Errorf("vdl: yyParse returned error code %d", errCode)
}
return lex.exprs
}
// ExtractExprPackagePaths returns any package paths that appear in named constants
// in expr. i.e. "a/b/c".Foo => "a/b/c".
func ExtractExprPackagePaths(expr ConstExpr) []string {
var paths []string
switch e := expr.(type) {
case *ConstNamed:
if path := packageFromName(e.Name); len(path) > 0 {
paths = append(paths, path)
}
case *ConstCompositeLit:
for _, kv := range e.KVList {
paths = append(paths, ExtractExprPackagePaths(kv.Key)...)
paths = append(paths, ExtractExprPackagePaths(kv.Value)...)
}
paths = append(paths, ExtractTypePackagePaths(e.Type)...)
case *ConstIndexed:
paths = append(paths, ExtractExprPackagePaths(e.Expr)...)
paths = append(paths, ExtractExprPackagePaths(e.IndexExpr)...)
case *ConstTypeConv:
paths = append(paths, ExtractTypePackagePaths(e.Type)...)
paths = append(paths, ExtractExprPackagePaths(e.Expr)...)
case *ConstTypeObject:
paths = append(paths, ExtractTypePackagePaths(e.Type)...)
case *ConstBinaryOp:
paths = append(paths, ExtractExprPackagePaths(e.Lexpr)...)
paths = append(paths, ExtractExprPackagePaths(e.Rexpr)...)
case *ConstUnaryOp:
paths = append(paths, ExtractExprPackagePaths(e.Expr)...)
default:
// leaf expression with no embedded expressions or types.
}
return paths
}
func ExtractTypePackagePaths(typ Type) []string {
var paths []string
switch t := typ.(type) {
case *TypeNamed:
if path := packageFromName(t.Name); len(path) > 0 {
paths = append(paths, path)
}
case *TypeArray:
paths = append(paths, ExtractTypePackagePaths(t.Elem)...)
case *TypeList:
paths = append(paths, ExtractTypePackagePaths(t.Elem)...)
case *TypeSet:
paths = append(paths, ExtractTypePackagePaths(t.Key)...)
case *TypeMap:
paths = append(paths, ExtractTypePackagePaths(t.Key)...)
paths = append(paths, ExtractTypePackagePaths(t.Elem)...)
case *TypeStruct:
for _, f := range t.Fields {
paths = append(paths, ExtractTypePackagePaths(f.Type)...)
}
case *TypeUnion:
for _, f := range t.Fields {
paths = append(paths, ExtractTypePackagePaths(f.Type)...)
}
case *TypeOptional:
paths = append(paths, ExtractTypePackagePaths(t.Base)...)
default:
// leaf type with no embedded types.
}
return paths
}
func packageFromName(name string) string {
if strings.HasPrefix(name, `"`) {
if parts := strings.SplitN(name[1:], `".`, 2); len(parts) == 2 {
return parts[0]
}
}
return ""
}
// lexer implements the yyLexer interface for the yacc-generated parser.
//
// An oddity: lexer also holds the result of the parse. Most yacc examples hold
// parse results in package-scoped (global) variables, but doing that would mean
// we wouldn't be able to run separate parses concurrently. To enable that we'd
// need each invocation of yyParse to mutate its own result, but unfortunately
// the Go yacc tool doesn't provide any way to pass extra arguments to yyParse.
//
// So we cheat and hold the parse result in the lexer, and in the yacc rules we
// call lexVDLFile(yylex) to convert from the yyLexer interface back to the
// concrete lexer type, and retrieve a pointer to the parse result.
type lexer struct {
// Fields for lexing / scanning the input source file.
name string
scanner scanner.Scanner
errs *vdlutil.Errors
startTok int // One of our dummy start tokens.
started bool // Has the dummy start token already been emitted?
sawEOF bool // Have we already seen the end-of-file?
prevTok token // Previous token, used for auto-semicolons and errors.
// Fields holding the result of file and config parsing.
comments commentMap
vdlFile *File
// Field holding the result of expr parsing.
exprs []ConstExpr
}
func newLexer(fileName string, src io.Reader, startTok int, errs *vdlutil.Errors) *lexer {
l := &lexer{name: fileName, errs: errs, startTok: startTok, vdlFile: &File{BaseName: path.Base(fileName)}}
l.comments.init()
l.scanner.Init(src)
// Don't produce character literal tokens, but do scan comments.
l.scanner.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
// Don't treat '\n' as whitespace, so we can auto-insert semicolons.
l.scanner.Whitespace = 1<<'\t' | 1<<'\r' | 1<<' '
l.scanner.Error = func(s *scanner.Scanner, msg string) {
l.Error(msg)
}
return l
}
type token struct {
t rune
text string
pos Pos
}
func (t token) String() string {
return fmt.Sprintf("%v %U %s", t.pos, t.t, t.text)
}
// The lex* functions below all convert the yyLexer input arg into a concrete
// lexer as their first step. The type conversion is always safe since we're
// the ones who called yyParse, and thus know the concrete type is always lexer.
// lexVDLFile retrieves the File parse result from the yyLexer interface. This
// is called in the yacc rules to fill in the parse result.
func lexVDLFile(yylex yyLexer) *File {
return yylex.(*lexer).vdlFile
}
// lexPosErrorf adds an error with positional information, on a type
// implementing the yyLexer interface. This is called in the yacc rules to
// throw errors.
func lexPosErrorf(yylex yyLexer, pos Pos, format string, v ...interface{}) {
yylex.(*lexer).posErrorf(pos, format, v...)
}
// lexGenEOF tells the lexer to generate EOF tokens from now on, as if the end
// of file had been seen. This is called in the yacc rules to terminate the
// parse even if the file still has tokens.
func lexGenEOF(yylex yyLexer) {
yylex.(*lexer).sawEOF = true
}
// lexStoreExprs stores the parsed exprs in the lexer.
func lexStoreExprs(yylex yyLexer, exprs []ConstExpr) {
yylex.(*lexer).exprs = exprs
}
var keywords = map[string]int{
"const": tCONST,
"enum": tENUM,
"error": tERROR,
"import": tIMPORT,
"interface": tINTERFACE,
"map": tMAP,
"package": tPACKAGE,
"set": tSET,
"stream": tSTREAM,
"struct": tSTRUCT,
"type": tTYPE,
"typeobject": tTYPEOBJECT,
"union": tUNION,
}
type nextRune struct {
t rune
id int
}
// knownPunct is a map of our known punctuation. We support 1 and 2 rune
// combinations, where 2 rune combos must be immediately adjacent with no
// intervening whitespace. The 2-rune combos always take precedence over the
// 1-rune combos. Every entry is a valid 1-rune combo, which is returned as-is
// without a special token id; the ascii value represents itself.
var knownPunct = map[rune][]nextRune{
';': nil,
':': nil,
',': nil,
'.': nil,
'*': nil,
'(': nil,
')': nil,
'[': nil,
']': nil,
'{': nil,
'}': nil,
'+': nil,
'-': nil,
'/': nil,
'%': nil,
'^': nil,
'?': nil,
'!': {{'=', tNE}},
'=': {{'=', tEQEQ}},
'<': {{'=', tLE}, {'<', tLSH}},
'>': {{'=', tGE}, {'>', tRSH}},
'|': {{'|', tOROR}},
'&': {{'&', tANDAND}},
}
// autoSemi determines whether to automatically add a semicolon, based on the
// rule that semicolons are always added at the end of each line after certain
// tokens. The Go auto-semicolon rule is described here:
// http://golang.org/ref/spec#Semicolons
func autoSemi(prevTok token) bool {
return prevAutoSemi[prevTok.t] && prevTok.pos.IsValid()
}
var prevAutoSemi = map[rune]bool{
scanner.Ident: true,
scanner.Int: true,
scanner.Float: true,
scanner.String: true,
scanner.RawString: true,
')': true,
']': true,
'}': true,
'>': true,
}
const yaccEOF int = 0 // yacc interprets 0 as the end-of-file marker
func init() {
// yyDebug is defined in the yacc-generated grammar.go file. Setting it to 1
// only produces output on syntax errors; set it to 4 to generate full debug
// output. Sadly yacc doesn't give position information describing the error.
yyDebug = 1
}
// A note on the comment-tracking strategy. During lexing we generate
// commentBlocks, defined as a sequence of adjacent or abutting comments (either
// // or /**/) with no intervening tokens. Adjacent means that the previous
// comment ends on the line immediately before the next one starts, and abutting
// means that the previous comment ends on the same line as the next one starts.
//
// At the end of the parse we try to attach comment blocks to parse tree items.
// We use a heuristic that works for common cases, but isn't perfect - it
// mis-associates some styles of comments, and we don't ensure all comment
// blocks will be associated to an item.
type commentBlock struct {
text string
firstLine int
lastLine int
}
// update returns true and adds tok to this block if tok is adjacent or
// abutting, otherwise it returns false without mutating the block. Since we're
// handling newlines explicitly in the lexer, we never get comment tokens with
// trailing newlines. We can get embedded newlines via /**/ style comments.
func (cb *commentBlock) update(tok token) bool {
if cb.text == "" {
// First update in this block.
cb.text = tok.text
cb.firstLine = tok.pos.Line
cb.lastLine = tok.pos.Line + strings.Count(tok.text, "\n")
return true
}
if cb.lastLine >= tok.pos.Line-1 {
// The tok is adjacent or abutting.
if cb.lastLine == tok.pos.Line-1 {
// The tok is adjacent - need a newline.
cb.text += "\n"
cb.lastLine++
}
cb.text += tok.text
cb.lastLine += strings.Count(tok.text, "\n")
return true
}
return false
}
// commentMap keeps track of blocks of comments in a file. We store comment
// blocks in maps by first line, and by last line. Note that technically there
// could be more than one commentBlock ending on the same line, due to /**/
// style comments. We ignore this rare case and just keep the first one.
type commentMap struct {
byFirst map[int]commentBlock
byLast map[int]commentBlock
cur commentBlock
prevTokenPos Pos
}
func (cm *commentMap) init() {
cm.byFirst = make(map[int]commentBlock)
cm.byLast = make(map[int]commentBlock)
}
// addComment adds a comment token to the map, either appending to the current
// block or ending the current block and starting a new one.
func (cm *commentMap) addComment(tok token) {
if !cm.cur.update(tok) {
cm.endBlock()
if !cm.cur.update(tok) {
panic(fmt.Errorf("vdl: couldn't update current comment block with token %v", tok))
}
}
// Here's an example of why we need the special case endBlock logic.
//
// type Foo struct {
// // doc1
// A int // doc2
// // doc3
// B int
// }
//
// The problem is that without the special-case, we'd group doc2 and doc3
// together into the same block. That may actually be correct some times, but
// it's more common for doc3 to be semantically associated with field B. Thus
// if we've already seen any token on the same line as this comment block, we
// end the block immediately. This means that comments appearing on the same
// line as any other token are forced to be a single comment block.
if cm.prevTokenPos.Line == tok.pos.Line {
cm.endBlock()
}
}
func (cm *commentMap) handleToken(tok token) {
cm.endBlock()
cm.prevTokenPos = tok.pos
}
// endBlock adds the the current comment block to the map, and resets it in
// preparation for new comments to be added. In the rare case where we see
// comment blocks that either start or end on the same line, we just keep the
// first comment block that was inserted.
func (cm *commentMap) endBlock() {
_, inFirst := cm.byFirst[cm.cur.firstLine]
_, inLast := cm.byLast[cm.cur.lastLine]
if cm.cur.text != "" && !inFirst && !inLast {
cm.byFirst[cm.cur.firstLine] = cm.cur
cm.byLast[cm.cur.lastLine] = cm.cur
}
cm.cur.text = ""
cm.cur.firstLine = 0
cm.cur.lastLine = 0
}
// getDoc returns the documentation string associated with pos. Our rule is the
// last line of the documentation must end on the line immediately before pos.
// Once a comment block has been returned it isn't eligible to be attached to
// any other item, and is deleted from the map.
//
// The returned string is either empty, or is newline terminated.
func (cm *commentMap) getDoc(pos Pos) string {
block := cm.byLast[pos.Line-1]
if block.text == "" {
return ""
}
doc := block.text + "\n"
delete(cm.byFirst, block.firstLine)
delete(cm.byLast, block.lastLine)
return doc
}
// getDocSuffix returns the suffix documentation associated with pos. Our rule
// is the first line of the documentation must be on the same line as pos. Once
// a comment block has been returned it isn't eligible to be attached to any
// other item, and is deleted from the map.
//
// The returned string is either empty, or has a leading space.
func (cm *commentMap) getDocSuffix(pos Pos) string {
block := cm.byFirst[pos.Line]
if block.text == "" {
return ""
}
doc := " " + block.text
delete(cm.byFirst, block.firstLine)
delete(cm.byLast, block.lastLine)
return doc
}
// getFileDoc returns the file documentation. Our rule is that the first line
// of the documentation must occur on the first line of the file, and all other
// comments must have already been attached. Once a comment block has been
// returned it isn't eligible to be attached to any other item, and is deleted
// from the map.
//
// The returned string is either empty, or is newline terminated.
func (cm *commentMap) getFileDoc() string {
block := cm.byFirst[1]
if block.text == "" {
return ""
}
doc := block.text + "\n"
delete(cm.byFirst, block.firstLine)
delete(cm.byLast, block.lastLine)
return doc
}
func attachTypeComments(t Type, cm *commentMap, suffix bool) {
switch tu := t.(type) {
case *TypeEnum:
for _, label := range tu.Labels {
if suffix {
label.DocSuffix = cm.getDocSuffix(label.Pos)
} else {
label.Doc = cm.getDoc(label.Pos)
}
}
case *TypeArray:
attachTypeComments(tu.Elem, cm, suffix)
case *TypeList:
attachTypeComments(tu.Elem, cm, suffix)
case *TypeSet:
attachTypeComments(tu.Key, cm, suffix)
case *TypeMap:
attachTypeComments(tu.Key, cm, suffix)
attachTypeComments(tu.Elem, cm, suffix)
case *TypeStruct:
for _, field := range tu.Fields {
if suffix {
field.DocSuffix = cm.getDocSuffix(field.Pos)
} else {
field.Doc = cm.getDoc(field.Pos)
}
attachTypeComments(field.Type, cm, suffix)
}
case *TypeUnion:
for _, field := range tu.Fields {
if suffix {
field.DocSuffix = cm.getDocSuffix(field.Pos)
} else {
field.Doc = cm.getDoc(field.Pos)
}
attachTypeComments(field.Type, cm, suffix)
}
case *TypeOptional:
attachTypeComments(tu.Base, cm, suffix)
case *TypeNamed:
// Terminate the recursion at named types.
default:
panic(fmt.Errorf("vdl: unhandled type %#v", t))
}
}
// attachComments causes all comments collected during the parse to be attached
// to the appropriate parse tree items. This should only be called after the
// parse has completed.
func (l *lexer) attachComments() {
f := l.vdlFile
// First attach all suffix docs - these occur on the same line.
f.PackageDef.DocSuffix = l.comments.getDocSuffix(f.PackageDef.Pos)
for _, x := range f.Imports {
x.DocSuffix = l.comments.getDocSuffix(x.Pos)
}
for _, x := range f.ErrorDefs {
x.DocSuffix = l.comments.getDocSuffix(x.Pos)
}
for _, x := range f.TypeDefs {
x.DocSuffix = l.comments.getDocSuffix(x.Pos)
attachTypeComments(x.Type, &l.comments, true)
}
for _, x := range f.ConstDefs {
x.DocSuffix = l.comments.getDocSuffix(x.Pos)
}
for _, x := range f.Interfaces {
x.DocSuffix = l.comments.getDocSuffix(x.Pos)
for _, y := range x.Embeds {
y.DocSuffix = l.comments.getDocSuffix(y.Pos)
}
for _, y := range x.Methods {
y.DocSuffix = l.comments.getDocSuffix(y.Pos)
}
}
// Now attach the docs - these occur on the line immediately before.
f.PackageDef.Doc = l.comments.getDoc(f.PackageDef.Pos)
for _, x := range f.Imports {
x.Doc = l.comments.getDoc(x.Pos)
}
for _, x := range f.ErrorDefs {
x.Doc = l.comments.getDoc(x.Pos)
}
for _, x := range f.TypeDefs {
x.Doc = l.comments.getDoc(x.Pos)
attachTypeComments(x.Type, &l.comments, false)
}
for _, x := range f.ConstDefs {
x.Doc = l.comments.getDoc(x.Pos)
}
for _, x := range f.Interfaces {
x.Doc = l.comments.getDoc(x.Pos)
for _, y := range x.Embeds {
y.Doc = l.comments.getDoc(y.Pos)
}
for _, y := range x.Methods {
y.Doc = l.comments.getDoc(y.Pos)
}
}
// Finally attach the top-level file doc - this occurs on the first line.
f.Doc = l.comments.getFileDoc()
}
// nextToken uses the text/scanner package to scan the input for the next token.
func (l *lexer) nextToken() (tok token) {
tok.t = l.scanner.Scan()
tok.text = l.scanner.TokenText()
// Both Pos and scanner.Position start line and column numbering at 1.
tok.pos = Pos{Line: l.scanner.Position.Line, Col: l.scanner.Position.Column}
return
}
// handleImag handles imaginary literals "[number]i" by peeking ahead.
func (l *lexer) handleImag(tok token, lval *yySymType) bool {
if l.scanner.Peek() != 'i' {
return false
}
l.scanner.Next()
rat := new(big.Rat)
if _, ok := rat.SetString(tok.text); !ok {
l.posErrorf(tok.pos, "can't convert token [%v] to imaginary literal", tok)
}
lval.imagpos.pos = tok.pos
lval.imagpos.imag = (*BigImag)(rat)
return true
}
// translateToken takes the token we just scanned, and translates it into a
// token usable by yacc (lval and id). The done return arg is true when a real
// yacc token was generated, or false if we need another next/translate pass.
func (l *lexer) translateToken(tok token, lval *yySymType) (id int, done bool) {
switch tok.t {
case scanner.EOF:
l.sawEOF = true
if autoSemi(l.prevTok) {
return ';', true
}
return yaccEOF, true
case '\n':
if autoSemi(l.prevTok) {
return ';', true
}
// Returning done=false ensures next/translate will be called again so that
// this newline is skipped; id=yaccEOF is a dummy value that's ignored.
return yaccEOF, false
case scanner.String, scanner.RawString:
var err error
lval.strpos.Pos = tok.pos
lval.strpos.String, err = strconv.Unquote(tok.text)
if err != nil {
l.posErrorf(tok.pos, "can't convert token [%v] to string literal", tok)
}
return tSTRLIT, true
case scanner.Int:
if l.handleImag(tok, lval) {
return tIMAGLIT, true
}
lval.intpos.pos = tok.pos
lval.intpos.int = new(big.Int)
if _, ok := lval.intpos.int.SetString(tok.text, 0); !ok {
l.posErrorf(tok.pos, "can't convert token [%v] to integer literal", tok)
}
return tINTLIT, true
case scanner.Float:
if l.handleImag(tok, lval) {
return tIMAGLIT, true
}
lval.ratpos.pos = tok.pos
lval.ratpos.rat = new(big.Rat)
if _, ok := lval.ratpos.rat.SetString(tok.text); !ok {
l.posErrorf(tok.pos, "can't convert token [%v] to float literal", tok)
}
return tRATLIT, true
case scanner.Ident:
// Either the identifier is a known keyword, or we pass it through as IDENT.
if keytok, ok := keywords[tok.text]; ok {
lval.pos = tok.pos
return keytok, true
}
lval.strpos.Pos = tok.pos
lval.strpos.String = tok.text
return tIDENT, true
case scanner.Comment:
l.comments.addComment(tok)
// Comments aren't considered tokens, just like the '\n' case.
return yaccEOF, false
default:
// Either the rune is in our known punctuation whitelist, or we've hit a
// syntax error.
if nextRunes, ok := knownPunct[tok.t]; ok {
// Peek at the next rune and compare against our list of next runes. If
// we find a match we return the id in next, otherwise just return the
// original rune. This means that 2-rune tokens always take precedence
// over 1-rune tokens. Either way the pos is set to the original rune.
lval.pos = tok.pos
peek := l.scanner.Peek()
for _, next := range nextRunes {
if peek == next.t {
l.scanner.Next()
return next.id, true
}
}
return int(tok.t), true
}
l.posErrorf(tok.pos, "unexpected token [%v]", tok)
l.sawEOF = true
return yaccEOF, true
}
}
// Lex is part of the yyLexer interface, called by the yacc-generated parser.
func (l *lexer) Lex(lval *yySymType) int {
// Emit a dummy start token indicating what type of parse we're performing.
if !l.started {
l.started = true
switch l.startTok {
case startFileImports, startFile, startConfigImports, startConfig, startExprs:
return l.startTok
default:
panic(fmt.Errorf("vdl: unhandled parse start token %d", l.startTok))
}
}
// Always return EOF after we've scanned it. This ensures we emit EOF on the
// next Lex call after scanning EOF and adding an auto-semicolon.
if l.sawEOF {
return yaccEOF
}
// Run next/translate in a loop to handle newline-triggered auto-semicolons;
// nextToken needs to generate newline tokens so that we can trigger the
// auto-semicolon logic, but if the newline doesn't generate an auto-semicolon
// we should skip the token and move on to the next one.
for {
tok := l.nextToken()
if id, done := l.translateToken(tok, lval); done {
l.prevTok = tok
l.comments.handleToken(tok)
return id
}
}
}
// Error is part of the yyLexer interface, called by the yacc-generated parser.
// Unfortunately yacc doesn't give good error information - we dump the position
// of the previous scanned token as an approximation of where the error is.
func (l *lexer) Error(s string) {
l.posErrorf(l.prevTok.pos, "%s", s)
}
// posErrorf generates an error with file and pos info.
func (l *lexer) posErrorf(pos Pos, format string, v ...interface{}) {
var posstr string
if pos.IsValid() {
posstr = pos.String()
}
l.errs.Errorf(l.name+":"+posstr+" "+format, v...)
}