local tokens = {
	whitespace = {
		[" "] = true,
		["\t"] = true,
	},
	line = {
		["\n"] = true,
	},
	keyword = {
		["gebruik"] = true,
		["zolang"] = true,
		["als"] = true,
		["anders"] = true,
	},
	initStatement = {
		["gebruik"] = true,
	},
	initVars = {
		["zwOog"] = true,
		["kleurOog"] = true,
		["kompas"] = true,
	},
	func = {
		["stapVooruit"] = true,
		["stapAchteruit"] = true,
		["draaiLinks"] = true,
		["draaiRechts"] = true
	},
	ident = {},
	digits = {},
	operator = {
		["*"] = true,
		["-"] = true,
		["+"] = true,
		["%"] = true,
		["/"] = true,
	},
	comparison = {
		["=="] = true,
		["!="] = true,
		["<"] = true,
		[">"] = true,
	},
	whileStatement = {
		["zolang"] = true,
	},
	ifStatement = {
		["als"] = true,
	},
	elseStatement = {
		["anders"] = true,
	}
}

for c=("a"):byte(), ("z"):byte() do
	tokens.ident[string.char(c)] = true
end

for c=("0"):byte(), ("9"):byte() do
	tokens.ident[string.char(c)] = true
end

--[[local function lex(text)
	local char = 1
	local function get(p)
		local tPos = p or char
		return text:sub(tPos,tPos)
	end
	local beginline = true
	local isProperty = true
	local data = {{}}
	local line = data[1]
	local posOffset = 0
	local function newline()
		posOffset = char
		table.insert(data,{})
		line = data[#data]
	end
	local function addData(txt,type)
		if line[#line] and line[#line].type == type then
			line[#line].data = line[#line].data..txt
			line[#line].posLast = line[#line].posLast+#txt
		else
			table.insert(line,{data=txt,posFirst=(char-posOffset),posLast=(char-posOffset)+(#txt-1),type=type})
		end
	end
	while char <= #text do
		local txt = get()
		addData(txt, "ident")
		if txt == "\n" then
			newline()
			beginline = true
			isProperty = true
		end
		--write(txt)
		char = char+1
	end
	return data
end]]

local function lex(text)
	local pos = 1
	local function get(p)
		local tPos = p or pos
		return text:sub(tPos,tPos)
	end
	local beginline = true
	local isProperty = true
	local data = {{}}
	local line = data[1]
	local ln = 1
	local posOffset = 0
	local function newline()
		posOffset = pos
		table.insert(data,{})
		line = data[#data]
		ln = ln+1
	end
	local function addData(txt,type)
		if line[#line] and line[#line].type == type then
			line[#line].data = line[#line].data..txt
			line[#line].posLast = line[#line].posLast+#txt
		else
			table.insert(line,{data=txt,posFirst=pos-#txt,posLast=pos-1,type=type})
		end
	end
	--[[while char <= #text do
		local txt = get()
		addData(txt, "ident")
		if txt == "\n" then
			newline()
			beginline = true
			isProperty = true
		end
		--write(txt)
		char = char+1
	end]]

	local function getToken()
		if pos > #text then
			return nil
		end
		local c = text:sub(pos,pos)
		local t = c
		local b,e = pos,pos
		if c:find("[a-z]") then
			b,e = text:find("[a-zA-Z]+", pos)
		elseif c:find("[0-9]") then
			b,e = text:find("[0-9]+", pos)
		elseif not tokens.line[t] and not tokens.whitespace[t] then
			b,e = text:find("%S+",pos)
		end
		t = text:sub(b,e)
		pos = e+1
		--print(b..".."..e..": "..t)
		return t
	end

	token = getToken()
	while token do
		if tokens.whitespace[token] then
			addData(token,"whitespace")

		elseif tokens.keyword[token] then
			addData(token,"keyword")

		elseif token:match("[0-9]+") == token then
			addData(token,"value")

		elseif tokens.operator[token] or tokens.comparison[token] then
			addData(token,"operator")

		elseif tokens.initVars[token] or tokens.func[token] then
			addData(token, "function")

		elseif tokens.line[token] then
			newline()

		else
			addData(token, "ident")
		end
		token = getToken()
	end
	return data
end

return lex