Change OvaleLexer into an object factory to generate lexers.
Johnny C. Lam [07-13-14 - 11:31]
Change OvaleLexer into an object factory to generate lexers.
The calling code must provide an iterator, which is typically generated
with OvaleLexer.scan. The lexer buffers the token stream so that it may
be accessed with look-ahead capabilities.
The lexer provides two methods for accessing the buffered token stream:
Peek(k) buffers the next k tokens from the stream and returns the k'th one.
Consume(k) consumes the next k tokens from the buffer and returns the k'th one.
This allows the following code to iterate through the token stream without
consuming extra tokens.
-- Process tokens until seeing a '}'.
local tokenType, token = lexer:Peek()
while tokenType ~= "}" do
lexer:Consume()
-- do stuff here with the token
tokenType, token = lexer:Peek()
end
-- Here, the next token to be consumed is still '}'.
git-svn-id: svn://svn.curseforge.net/wow/ovale/mainline/trunk@1540 d5049fe3-3747-40f7-a4b5-f36d6801af5f
diff --git a/Ovale.toc b/Ovale.toc
index fcb93bb..719a772 100644
--- a/Ovale.toc
+++ b/Ovale.toc
@@ -22,7 +22,6 @@ Ovale.lua
Profiler.lua
# Utility modules.
-OvaleLexer.lua
OvalePool.lua
OvalePoolGC.lua
OvalePoolRefCount.lua
@@ -42,6 +41,7 @@ OvaleEquipement.lua
OvaleFuture.lua
OvaleGUID.lua
OvaleLatency.lua
+OvaleLexer.lua
OvaleOptions.lua
OvalePaperDoll.lua
OvalePower.lua
diff --git a/OvaleLexer.lua b/OvaleLexer.lua
index 2370005..344247c 100644
--- a/OvaleLexer.lua
+++ b/OvaleLexer.lua
@@ -12,6 +12,9 @@ local OvaleLexer = {}
Ovale.OvaleLexer = OvaleLexer
--<private-static-properties>
+local OvaleQueue = Ovale.OvaleQueue
+local setmetatable = setmetatable
+
-- Additional Lua functions used by Penlight pl.lexer module.
local error = error
local type = type
@@ -499,5 +502,78 @@ end
--</vendor-code>
--<public-static-properties>
+OvaleLexer.typeQueue = nil
+OvaleLexer.tokenQueue = nil
+OvaleLexer.iterator = nil
+OvaleLexer.endOfStream = nil
OvaleLexer.scan = lexer.scan
+OvaleLexer.__index = OvaleLexer
--</public-static-properties>
+
+--<public-static-methods>
+do
+ -- Class constructor
+ setmetatable(OvaleLexer, { __call = function(self, ...) return self:New(...) end })
+end
+
+function OvaleLexer:New(name, iterator)
+ local obj = {
+ typeQueue = OvaleQueue:NewDeque(name .. "_typeQueue"),
+ tokenQueue = OvaleQueue:NewDeque(name .. "_tokenQueue"),
+ iterator = iterator,
+ }
+ return setmetatable(obj, self)
+end
+
+function OvaleLexer:Release()
+ for key in pairs(self) do
+ self[key] = nil
+ end
+end
+
+function OvaleLexer:Consume(index)
+ index = index or 1
+ local tokenType, token
+ -- Consume the tokens in the buffer first.
+ while index > 0 and self.typeQueue:Size() > 0 do
+ tokenType = self.typeQueue:RemoveFront()
+ token = self.tokenQueue:RemoveFront()
+ if not tokenType then
+ break
+ end
+ index = index - 1
+ end
+ -- Consume from the token stream if needed.
+ while index > 0 do
+ tokenType, token = self.iterator()
+ if not tokenType then
+ break
+ end
+ index = index - 1
+ end
+ return tokenType, token
+end
+
+function OvaleLexer:Peek(index)
+ index = index or 1
+ local tokenType, token
+ while index > self.typeQueue:Size() do
+ if self.endOfStream then
+ break
+ else
+ tokenType, token = self.iterator()
+ if not tokenType then
+ self.endOfStream = true
+ break
+ end
+ self.typeQueue:InsertBack(tokenType)
+ self.tokenQueue:InsertBack(token)
+ end
+ end
+ if index <= self.typeQueue:Size() then
+ tokenType = self.typeQueue:At(index)
+ token = self.tokenQueue:At(index)
+ end
+ return tokenType, token
+end
+--</public-static-methods>