Handle the data changing levels gracefully

diff --git a/get-live-data.lua b/get-live-data.lua
index c4c8458..1050537 100644
--- a/get-live-data.lua
+++ b/get-live-data.lua
@@ -1,3 +1,8 @@
+local ITEM_LINK_TEMPLATE = "^item:(%d+):0:0:0:0:0:0:0:"
+local ITEM_LINK_LEVEL_TEMPLATE = ITEM_LINK_TEMPLATE .. "(%d+)"
+local SUFFIX_LINK_TEMPLATE = "^item:11993:0:0:0:0:0:(%-?%d+):100:"
+local SUFFIX_LINK_LEVEL_TEMPLATE = SUFFIX_LINK_TEMPLATE .. "(%d+)"
+
 local func = loadfile(...)

 loadfile("load-all-item-info.lua")()
@@ -17,19 +22,89 @@ IS_invalid_items = IS_status.items.invalid
 IS_invalid_suffixes = IS_status.suffixes.invalid
 IS_invalid_gems = IS_status.gems.invalid

+local itemIndex = {}
+local suffIndex = {}
+
+-- Find the max level scan for each item/suffix in the old data
+for k, v in pairs(item_info) do
+	local start, _, itemNum, level = k:find(ITEM_LINK_LEVEL_TEMPLATE)
+	if start then
+		itemNum = tonumber(itemNum)
+		level = tonumber(level)
+		itemIndex[itemNum] = itemIndex[itemNum] or { max = 0 }
+		itemIndex[itemNum][level] = k
+		if level > itemIndex[itemNum].max then
+			itemIndex[itemNum].max = level
+		end
+	else
+		local start, _, suffNum, level = k:find(SUFFIX_LINK_LEVEL_TEMPLATE)
+		if start then
+			suffNum = tonumber(suffNum)
+			level = tonumber(level)
+			suffIndex[suffNum] = suffIndex[suffNum] or { max = 0 }
+			suffIndex[suffNum][level] = k
+			if level > suffIndex[suffNum].max then
+				suffIndex[suffNum].max = level
+			end
+		end
+	end
+end
+
 --TODO: handle IS_gem_info too?
 for k, v in pairs(item_info) do
 	if IS_item_info[k] then
-		if #(v) > #(IS_item_info[k]) then
-			ItemsThatNeedManualIntervention[k] = IS_item_info[k]
-			IS_item_info[k] = v
+		local skip = false
+		local start, _, itemNum, level = k:find(ITEM_LINK_LEVEL_TEMPLATE)
+		if start then
+			itemNum = tonumber(itemNum)
+			level = tonumber(level)
+			if level >= itemIndex[itemNum].max then
+				for lvl, key in pairs(itemIndex[itemNum]) do
+					if type(lvl) == "number" then
+						if lvl < level then
+							IS_item_info[key] = nil
+							item_info[key] = nil
+							ItemsThatNeedManualIntervention[key] = nil
+						end
+					end
+				end
+			else
+				--TODO: print warning about less-than-max-level scan?
+				skip = true
+			end
 		else
-			ItemsThatNeedManualIntervention[k] = nil
+			local start, _, suffNum, level = k:find(SUFFIX_LINK_LEVEL_TEMPLATE)
+			if start then
+				suffNum = tonumber(suffNum)
+				level = tonumber(level)
+				if level >= suffIndex[suffNum].max then
+					for lvl, key in pairs(suffIndex[suffNum]) do
+						if type(lvl) == "number" then
+							if lvl < level then
+								IS_item_info[key] = nil
+								item_info[key] = nil
+								ItemsThatNeedManualIntervention[key] = nil
+							end
+						end
+					end
+				else
+					--TODO: print warning about less-than-max-level scan?
+					skip = true
+				end
+			end
+		end
+		if not skip then
+			if #(v) > #(IS_item_info[k]) then
+				ItemsThatNeedManualIntervention[k] = IS_item_info[k]
+				IS_item_info[k] = v
+			else
+				ItemsThatNeedManualIntervention[k] = nil
+			end
 		end
 	elseif type(k) ~= "string" then
 		ItemsThatNeedManualIntervention[k] = string.format("Unknown item link format: %q", tostring(k))
 	else
-		local start, _, itemNum = k:find("^item:(%d+):0:0:0:0:0:0:0:")
+		local start, _, itemNum = k:find(ITEM_LINK_TEMPLATE)
 		if start then
 			itemNum = tonumber(itemNum)
 			if type(IS_invalid_items[itemNum]) == "number" then
@@ -42,9 +117,18 @@ for k, v in pairs(item_info) do
 			elseif IS_invalid_items[itemNum] ~= false and IS_invalid_items[itemNum] ~= nil then
 				ItemsThatNeedManualIntervention[k] = string.format("Unhandled invalid item type: %s (tostring = %q)", type(IS_invalid_items[itemNum]), tostring(IS_invalid_items[itemNum]))
 			end
+			for level, key in pairs(itemIndex[itemNum]) do
+				if level == itemIndex[itemNum].max then
+					v = item_info[itemIndex[itemNum][itemIndex[itemNum].max]]
+				else
+					IS_item_info[key] = nil
+					item_info[key] = nil
+					ItemsThatNeedManualIntervention[key] = nil
+				end
+			end
 			IS_item_info[k] = v
 		else
-			local start, _, suffNum = k:find("^item:11993:0:0:0:0:0:(%-?%d+):100:")
+			local start, _, suffNum = k:find(SUFFIX_LINK_TEMPLATE)
 			if start then
 				suffNum = tonumber(suffNum)
 				if type(IS_invalid_suffixes[suffNum]) == "number" then
@@ -57,6 +141,15 @@ for k, v in pairs(item_info) do
 				elseif IS_invalid_suffixes[suffNum] ~= false and IS_invalid_suffixes[suffNum] ~= nil then
 					ItemsThatNeedManualIntervention[k] = string.format("Unhandled invalid suffix type: %s (tostring = %q)", type(IS_invalid_suffixes[suffNum]), tostring(IS_invalid_suffixes[suffNum]))
 				end
+				for level, key in pairs(suffIndex[suffNum]) do
+					if level == suffIndex[suffNum].max then
+						v = item_info[suffIndex[suffNum][suffIndex[suffNum].max]]
+					else
+						IS_item_info[key] = nil
+						item_info[key] = nil
+						ItemsThatNeedManualIntervention[key] = nil
+					end
+				end
 				IS_item_info[k] = v
 			else
 				ItemsThatNeedManualIntervention[k] = string.format("Unknown item link format: %q", k)