From 89af97dcc46b945f4ef7ad83efa3dcb4f4ab81d5 Mon Sep 17 00:00:00 2001
From: "Johnny C. Lam" <jlam88@gmail.com>
Date: Sun, 13 Jul 2014 11:31:09 +0000
Subject: [PATCH] Change OvaleLexer into an object factory to generate lexers.

The calling code must provide an iterator, which is typically generated
with OvaleLexer.scan.  The lexer buffers the token stream so that it may
be accessed with look-ahead capabilities.

The lexer provides two methods for accessing the buffered token stream:

Peek(k) buffers the next k tokens from the stream and returns the k'th one.
Consume(k) consumes the next k tokens from the buffer and returns the k'th one.

This allows the following code to iterate through the token stream without
consuming extra tokens.

    -- Process tokens until seeing a '}'.
    local tokenType, token = lexer:Peek()
    while tokenType ~= "}" do
        lexer:Consume()
        -- do stuff here with the token
        tokenType, token = lexer:Peek()
    end
    -- Here, the next token to be consumed is still '}'.

git-svn-id: svn://svn.curseforge.net/wow/ovale/mainline/trunk@1540 d5049fe3-3747-40f7-a4b5-f36d6801af5f
---
 Ovale.toc      |    2 +-
 OvaleLexer.lua |   76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/Ovale.toc b/Ovale.toc
index fcb93bb..719a772 100644
--- a/Ovale.toc
+++ b/Ovale.toc
@@ -22,7 +22,6 @@ Ovale.lua
 Profiler.lua
 
 # Utility modules.
-OvaleLexer.lua
 OvalePool.lua
 OvalePoolGC.lua
 OvalePoolRefCount.lua
@@ -42,6 +41,7 @@ OvaleEquipement.lua
 OvaleFuture.lua
 OvaleGUID.lua
 OvaleLatency.lua
+OvaleLexer.lua
 OvaleOptions.lua
 OvalePaperDoll.lua
 OvalePower.lua
diff --git a/OvaleLexer.lua b/OvaleLexer.lua
index 2370005..344247c 100644
--- a/OvaleLexer.lua
+++ b/OvaleLexer.lua
@@ -12,6 +12,9 @@ local OvaleLexer = {}
 Ovale.OvaleLexer = OvaleLexer
 
 --<private-static-properties>
+local OvaleQueue = Ovale.OvaleQueue
+local setmetatable = setmetatable
+
 -- Additional Lua functions used by Penlight pl.lexer module.
 local error = error
 local type = type
@@ -499,5 +502,78 @@ end
 --</vendor-code>
 
 --<public-static-properties>
+OvaleLexer.typeQueue = nil
+OvaleLexer.tokenQueue = nil
+OvaleLexer.iterator = nil
+OvaleLexer.endOfStream = nil
 OvaleLexer.scan = lexer.scan
+OvaleLexer.__index = OvaleLexer
 --</public-static-properties>
+
+--<public-static-methods>
+do
+	-- Class constructor
+	setmetatable(OvaleLexer, { __call = function(self, ...) return self:New(...) end })
+end
+
+function OvaleLexer:New(name, iterator)
+	local obj = {
+		typeQueue = OvaleQueue:NewDeque(name .. "_typeQueue"),
+		tokenQueue = OvaleQueue:NewDeque(name .. "_tokenQueue"),
+		iterator = iterator,
+	}
+	return setmetatable(obj, self)
+end
+
+function OvaleLexer:Release()
+	for key in pairs(self) do
+		self[key] = nil
+	end
+end
+
+function OvaleLexer:Consume(index)
+	index = index or 1
+	local tokenType, token
+	-- Consume the tokens in the buffer first.
+	while index > 0 and self.typeQueue:Size() > 0 do
+		tokenType = self.typeQueue:RemoveFront()
+		token = self.tokenQueue:RemoveFront()
+		if not tokenType then
+			break
+		end
+		index = index - 1
+	end
+	-- Consume from the token stream if needed.
+	while index > 0 do
+		tokenType, token = self.iterator()
+		if not tokenType then
+			break
+		end
+		index = index - 1
+	end
+	return tokenType, token
+end
+
+function OvaleLexer:Peek(index)
+	index = index or 1
+	local tokenType, token
+	while index > self.typeQueue:Size() do
+		if self.endOfStream then
+			break
+		else
+			tokenType, token = self.iterator()
+			if not tokenType then
+				self.endOfStream = true
+				break
+			end
+			self.typeQueue:InsertBack(tokenType)
+			self.tokenQueue:InsertBack(token)
+		end
+	end
+	if index <= self.typeQueue:Size() then
+		tokenType = self.typeQueue:At(index)
+		token = self.tokenQueue:At(index)
+	end
+	return tokenType, token
+end
+--</public-static-methods>
-- 
1.7.9.5