First public release

1 files changed, 186 insertions, 0 deletions

diff --git a/tokens/tokens.go b/tokens/tokens.go
new file mode 100644
index 0000000..e063730
--- /dev/null
+++ b/tokens/tokens.go
@@ -0,0 +1,186 @@
+package tokens
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type TokenId int
+
+func (t TokenId) String() string {
+	return tokenStr[t]
+}
+
+func (t TokenId) MarshalJSON() ([]byte, error) {
+	return json.Marshal(t.String())
+}
+
+const (
+	Eof TokenId = iota
+	Ident
+
+	None
+	True
+	False
+	Var
+	Const
+	Def
+	TNumber
+	TBool
+	TString
+	TFunc
+	TStruct
+	TArray
+	Return
+	If
+	Else
+	For
+	In
+	Continue
+	Break
+
+	Number
+	Char
+	String
+
+	Or
+	And
+	Eq
+	Ne
+	Ge
+	Le
+	BitShr
+	BitShl
+	TagE
+
+	LBrace
+	RBrace
+	LParen
+	RParen
+	LBracket
+	RBracket
+	Semicolon
+	Comma
+	Add
+	Sub
+	Mul
+	Mod
+	Div
+	Neg
+	Dot
+	Assign
+	Gt
+	Lt
+	Not
+	BitOr
+	BitAnd
+	BitXor
+	TestE
+)
+
+var (
+	tokenStr = []string{
+		"Eof",
+		"Ident",
+
+		// keywords
+		"none",
+		"True",
+		"False",
+		"Var",
+		"Const",
+		"Def",
+		"number",
+		"bool",
+		"string",
+		"func",
+		"struct",
+		"array",
+		"Return",
+		"If",
+		"Else",
+		"For",
+		"In",
+		"Continue",
+		"Break",
+
+		// literals
+		"Number",
+		"Character",
+		"String",
+
+		// double char
+		"Or",
+		"And",
+		"Eq",
+		"NE",
+		"Ge",
+		"Le",
+		"BitShr",
+		"BitShl",
+		"TagE",
+
+		// single character
+		"LBrace",
+		"RBrace",
+		"LParen",
+		"RParen",
+		"LBracket",
+		"RBracket",
+		"Semicolon",
+		"Comma",
+		"Add",
+		"Sub",
+		"Mul",
+		"Mod",
+		"Div",
+		"Neg",
+		"Dot",
+		"Assign",
+		"Gt",
+		"Lt",
+		"Not",
+		"BitOr",
+		"BitAnd",
+		"BitXor",
+		"TestE",
+	}
+)
+
+type Location struct {
+	File   string `json:"file"`
+	Line   int    `json:"line"`
+	Column int    `json:"col"`
+}
+
+func (l Location) String() string {
+	return fmt.Sprintf("%s:%d:%d", l.File, l.Line, l.Column)
+}
+
+func (l *Location) Inc() {
+	l.Column++
+}
+
+func (l *Location) Add(cols int) {
+	l.Column += cols
+}
+
+func (l *Location) Eol() {
+	l.Line++
+	l.Column = 1
+}
+
+type Token struct {
+	Id    TokenId  `json:"id"`
+	Loc   Location `json:"loc"`
+	Value string   `json:"val,omitempty"`
+}
+
+func (t Token) String() string {
+	switch t.Id {
+	case Ident:
+		return fmt.Sprintf("'%s'", t.Value)
+	default:
+		return fmt.Sprintf("%s at %s", t.Id, t.Loc)
+	}
+}