package tokens import ( "encoding/json" "fmt" ) type TokenId int func (t TokenId) String() string { return tokenStr[t] } func (t TokenId) MarshalJSON() ([]byte, error) { return json.Marshal(t.String()) } const ( Eof TokenId = iota Ident None True False Var Const Def TNumber TBool TString TFunc TStruct TArray Return If Else For In Continue Break Number Char String Or And Eq Ne Ge Le BitShr BitShl TagE LBrace RBrace LParen RParen LBracket RBracket Semicolon Comma Add Sub Mul Mod Div Neg Dot Assign Gt Lt Not BitOr BitAnd BitXor TestE ) var ( tokenStr = []string{ "Eof", "Ident", // keywords "none", "True", "False", "Var", "Const", "Def", "number", "bool", "string", "func", "struct", "array", "Return", "If", "Else", "For", "In", "Continue", "Break", // literals "Number", "Character", "String", // double char "Or", "And", "Eq", "NE", "Ge", "Le", "BitShr", "BitShl", "TagE", // single character "LBrace", "RBrace", "LParen", "RParen", "LBracket", "RBracket", "Semicolon", "Comma", "Add", "Sub", "Mul", "Mod", "Div", "Neg", "Dot", "Assign", "Gt", "Lt", "Not", "BitOr", "BitAnd", "BitXor", "TestE", } ) type Location struct { File string `json:"file"` Line int `json:"line"` Column int `json:"col"` } func (l Location) String() string { return fmt.Sprintf("%s:%d:%d", l.File, l.Line, l.Column) } func (l *Location) Inc() { l.Column++ } func (l *Location) Add(cols int) { l.Column += cols } func (l *Location) Eol() { l.Line++ l.Column = 1 } type Token struct { Id TokenId `json:"id"` Loc Location `json:"loc"` Value string `json:"val,omitempty"` } func (t Token) String() string { switch t.Id { case Ident: return fmt.Sprintf("'%s'", t.Value) default: return fmt.Sprintf("%s at %s", t.Id, t.Loc) } }