Added TES5Edit submodule, modified record filter to work from submodule, and generated records/flags file from submodule.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "reference/TES5Edit"]
+	path = reference/TES5Edit
+	url = https://github.com/TES5Edit/TES5Edit.git

reference/flag_filter.py

@@ -1,114 +0,0 @@
-import re
-import os
-import json
-from dataclasses import dataclass
-
-
-class Jsonify:
-    def encode(self, value=None):
-        if not value:
-            value = self.__dict__
-        return value
-
-@dataclass
-class Flag(Jsonify):
-    bit: int
-    description: str
-
-@dataclass
-class Record(Jsonify):
-    fourcc: str
-    name: str
-    flags: list[Flag]
-
-    
-re_flags = re.compile(".*?(\d+?),.*?'(.*)'")
-
-
-def find_records(data, re_record, re_first):
-    found_records = re_record.findall(data)
-
-    seen = set()
-    records = []
-    for record in found_records:
-        s = record.split('\n')
-        [first] = re_first.findall(s[0])
-        assert len(first) == 2, "Could not parse first line of record:\n{}".format(record)
-        flags = []
-        for line in s[2:-1]:
-            [f] = re_flags.findall(line)
-            assert len(f) == 2, "Could not parse flag definition: {}".format(line)
-            desc = f[1].replace("''", "'").replace("', '", "")
-            flags += [Flag(int(f[0]), desc)]
-        r = Record(first[0], first[1], flags)
-        if r.fourcc not in seen:
-            records += [r]
-            seen.add(r.fourcc)
-
-    return records
-
-
-def find_wbRecords(data):
-    re_wbRecord = re.compile("wbRecord\(.*?\n.*?wbFlagsList\(\[\n(?:.*?\n)*?.*?\]")
-    re_wbRecord_first = re.compile("wbRecord\((\w{4}).*?'(.+?)'.*?")
-    return find_records(data, re_wbRecord, re_wbRecord_first)
-
-
-def find_stat(data):
-    re_stat = re.compile("wbRecord\(STAT,.*?\n.*?\n(?:.*?\n)+?.*?]")
-    lines = re_stat.findall(data)[0].split('\n')
-    
-    re_stat_first = re.compile("STAT, '(.*?)'")
-    [name] = re_stat_first.findall(lines[0])
-    
-    re_stat_flag = re.compile("'(.*?)'")
-    flags = []
-    for i, line in enumerate(lines[2:-1]):
-        [l] = re_stat_flag.findall(line)
-        if l:
-            flags += [Flag(i, l)]
-        
-    return Record("STAT", name, flags)
-    
-    
-def find_concrete_wbRefRecords(data):
-    re_wbRefRecord = re.compile("wbRefRecord\(\w{4},.*?\n.*?wbFlagsList\(\[.*?\n(?:.*?\n)+?.*?\]")
-    re_wbRefRecord_first = re.compile("wbRefRecord\((\w{4}).*?'(.+?)'.*?")
-    return find_records(data, re_wbRefRecord, re_wbRefRecord_first)
-    
-
-def find_ReferenceRecords(data):
-    re_ReferenceRecordDef = re.compile("wbRefRecord\(aSignature, .*?\n(?:.*?\n)+?.*?]")
-    re_ReferenceRecordDef_first = re.compile("wbRefRecord\((\w+?), (\w+?),")
-    [d] = find_records(data, re_ReferenceRecordDef, re_ReferenceRecordDef_first)
-    
-    re_ReferenceRecord = re.compile("ReferenceRecord\((\w{4}), '(.*?)'\);")
-    rr = re_ReferenceRecord.findall(data)
-    
-    records = [Record(r[0], r[1], d.flags) for r in rr]
-    return records
-
-
-def find_wbRefRecords(data):
-    records = find_concrete_wbRefRecords(data)
-    records = find_ReferenceRecords(data)
-    return records
-
-
-fdir = os.path.dirname(__file__)
-with open(os.path.join(fdir, "wbDefinitionsTES5.pas"), "r") as f:
-    inp = f.read()
-
-records = find_wbRecords(inp) + [find_stat(inp)] + find_wbRefRecords(inp)
-records.sort(key = lambda x: x.fourcc)
-print(len(records))
-
-class Encoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, Jsonify):
-            return obj.encode()
-        else:
-            return json.JSONEncoder.default(self, obj)
-
-with open(os.path.join(fdir, "flags.json"), "w") as f:
-    json.dump(records, f, cls=Encoder, indent=4)

reference/flags.json

@@ -26,7 +26,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             }
         ]
     },
@@ -208,7 +208,7 @@
             },
             {
                 "bit": "19",
-                "description": "Can''t Wait"
+                "description": "Can't Wait"
             }
         ]
     },
@@ -782,7 +782,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -812,7 +812,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -842,7 +842,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -872,7 +872,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -912,7 +912,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -942,7 +942,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -972,7 +972,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -1007,7 +1007,7 @@
             },
             {
                 "bit": "29",
-                "description": "Don''t Havok Settle"
+                "description": "Don't Havok Settle"
             },
             {
                 "bit": "30",
@@ -1324,7 +1324,7 @@
             },
             {
                 "bit": "9",
-                "description": "ESL', '"
+                "description": "ESL"
             }
         ]
     },
@@ -1379,7 +1379,7 @@
         "flags": [
             {
                 "bit": "19",
-                "description": "Can''t Wait"
+                "description": "Can't Wait"
             }
         ]
     },

reference/record_filter.py

@@ -25,7 +25,7 @@ class Record(Jsonify):
 
 # open and read the records from xEdit source
 fdir = os.path.dirname(__file__)
-with open(os.path.join(fdir, "wbDefinitionsTES5.pas"), "r") as f:
+with open(os.path.join(fdir, "TES5Edit/Core/wbDefinitionsTES5.pas"), "r") as f:
     inp = f.read()
 
 # find all wbRecord/wbRefRecord calls
@@ -61,6 +61,8 @@ for string in all:
         assert len(flag_lines) > 0
         for line in flag_lines:
             bit, desc = re_flag_line.match(line).groups()
+            # fix '' pascal escape and ESL special case
+            desc = desc.replace("''", "'").replace("', '", "")
             flags += [Flag(bit, desc)]
     # handle STAT special case
     elif sig == "STAT":

reference/record_types.csv

@@ -1,137 +0,0 @@
-AACT,Action
-ACHR,Placed NPC
-ACTI,Activator
-ADDN,Addon Node
-ALCH,Ingestible
-AMMO,Ammunition
-ANIO,Animated Object
-APPA,Alchemical Apparatus
-ARMA,Armor Addon
-ARMO,Armor
-ARTO,Art Object
-ASPC,Acoustic Space
-ASTP,Association Type
-AVIF,Actor Value Information
-BOOK,Book
-BPTD,Body Part Data
-CAMS,Camera Shot
-CELL,Cell
-CLAS,Class
-CLDC,CLDC
-CLFM,Color
-CLMT,Climate
-COBJ,Constructible Object
-COLL,Collision Layer
-CONT,Container
-CPTH,Camera Path
-CSTY,Combat Style
-DEBR,Debris
-DIAL,Dialog Topic
-DLBR,Dialog Branch
-DLVW,Dialog View
-DOBJ,Default Object Manager
-DOOR,Door
-DUAL,Dual Cast Data
-ECZN,Encounter Zone
-EFSH,Effect Shader
-ENCH,Object Effect
-EQUP,Equip Type
-EXPL,Explosion
-EYES,Eyes
-FACT,Faction
-FLOR,Flora
-FLST,FormID List
-FSTP,Footstep
-FSTS,Footstep Set
-FURN,Furniture
-GLOB,Global
-GMST,Game Setting
-GRAS,Grass
-GRUP,Internal File Structure Group
-HAIR,HAIR
-HAZD,Hazard
-HDPT,Head Part
-IDLE,Idle Animation
-IDLM,Idle Marker
-IMAD,Image Space Adapter
-IMGS,Image Space
-INFO,Dialog response
-INGR,Ingredient
-IPCT,Impact
-IPDS,Impact Data Set
-KEYM,Key
-KYWD,Keyword
-LAND,Landscape
-LCRT,Location Reference Type
-LCTN,Location
-LENS,Lens Flare
-LGTM,Lighting Template
-LIGH,Light
-LSCR,Load Screen
-LTEX,Landscape Texture
-LVLI,Leveled Item
-LVLN,Leveled NPC
-LVSP,Leveled Spell
-MATO,Material Object
-MATT,Material Type
-MESG,Message
-MGEF,Magic Effect
-MISC,Misc. Item
-MOVT,Movement Type
-MSTT,Moveable Static
-MUSC,Music Type
-MUST,Music Track
-NAVI,Navigation Mesh Info Map
-NAVM,Navigation Mesh
-NOTE,NOTE
-NPC_,Non-Player Character (Actor)
-OTFT,Outfit
-PACK,Package
-PARW,Placed Arrow
-PBAR,Placed Barrier
-PBEA,Placed Beam
-PCON,Placed Cone/Voice
-PERK,Perk
-PFLA,Placed Flame
-PGRE,Placed Projectile
-PHZD,Placed Hazard
-PLYR,Player Reference
-PMIS,Placed Missile
-PROJ,Projectile
-PWAT,PWAT
-QUST,Quest
-RACE,Race
-REFR,Placed Object
-REGN,Region
-RELA,Relationship
-REVB,Reverb Parameters
-RFCT,Visual Effect
-RGDL,RGDL
-SCEN,Scene
-SCOL,SCOL
-SCOL,Static Collection
-SCPT,SCPT
-SCRL,Scroll
-SHOU,Shout
-SLGM,Soul Gem
-SMBN,Story Manager Branch Node
-SMEN,Story Manager Event Node
-SMQN,Story Manager Quest Node
-SNCT,Sound Category
-SNDR,Sound Descriptor
-SOPM,Sound Output Model
-SOUN,Sound Marker
-SPEL,Spell
-SPGD,Shader Particle Geometry
-STAT,Static
-TACT,Talking Activator
-TES4,Main File Header
-TREE,Tree
-TXST,Texture Set
-VOLI,Volumetric Lighting
-VTYP,Voice Type
-WATR,Water
-WEAP,Weapon
-WOOP,Word of Power
-WRLD,Worldspace
-WTHR,Weather