Add project files.

reference/flag_filter.py

@@ -0,0 +1,114 @@
+import re
+import os
+import json
+from dataclasses import dataclass
+
+
+class Jsonify:
+    def encode(self, value=None):
+        if not value:
+            value = self.__dict__
+        return value
+
+@dataclass
+class Flag(Jsonify):
+    bit: int
+    description: str
+
+@dataclass
+class Record(Jsonify):
+    fourcc: str
+    name: str
+    flags: list[Flag]
+
+    
+re_flags = re.compile(".*?(\d+?),.*?'(.*)'")
+
+
+def find_records(data, re_record, re_first):
+    found_records = re_record.findall(data)
+
+    seen = set()
+    records = []
+    for record in found_records:
+        s = record.split('\n')
+        [first] = re_first.findall(s[0])
+        assert len(first) == 2, "Could not parse first line of record:\n{}".format(record)
+        flags = []
+        for line in s[2:-1]:
+            [f] = re_flags.findall(line)
+            assert len(f) == 2, "Could not parse flag definition: {}".format(line)
+            desc = f[1].replace("''", "'").replace("', '", "")
+            flags += [Flag(int(f[0]), desc)]
+        r = Record(first[0], first[1], flags)
+        if r.fourcc not in seen:
+            records += [r]
+            seen.add(r.fourcc)
+
+    return records
+
+
+def find_wbRecords(data):
+    re_wbRecord = re.compile("wbRecord\(.*?\n.*?wbFlagsList\(\[\n(?:.*?\n)*?.*?\]")
+    re_wbRecord_first = re.compile("wbRecord\((\w{4}).*?'(.+?)'.*?")
+    return find_records(data, re_wbRecord, re_wbRecord_first)
+
+
+def find_stat(data):
+    re_stat = re.compile("wbRecord\(STAT,.*?\n.*?\n(?:.*?\n)+?.*?]")
+    lines = re_stat.findall(data)[0].split('\n')
+    
+    re_stat_first = re.compile("STAT, '(.*?)'")
+    [name] = re_stat_first.findall(lines[0])
+    
+    re_stat_flag = re.compile("'(.*?)'")
+    flags = []
+    for i, line in enumerate(lines[2:-1]):
+        [l] = re_stat_flag.findall(line)
+        if l:
+            flags += [Flag(i, l)]
+        
+    return Record("STAT", name, flags)
+    
+    
+def find_concrete_wbRefRecords(data):
+    re_wbRefRecord = re.compile("wbRefRecord\(\w{4},.*?\n.*?wbFlagsList\(\[.*?\n(?:.*?\n)+?.*?\]")
+    re_wbRefRecord_first = re.compile("wbRefRecord\((\w{4}).*?'(.+?)'.*?")
+    return find_records(data, re_wbRefRecord, re_wbRefRecord_first)
+    
+
+def find_ReferenceRecords(data):
+    re_ReferenceRecordDef = re.compile("wbRefRecord\(aSignature, .*?\n(?:.*?\n)+?.*?]")
+    re_ReferenceRecordDef_first = re.compile("wbRefRecord\((\w+?), (\w+?),")
+    [d] = find_records(data, re_ReferenceRecordDef, re_ReferenceRecordDef_first)
+    
+    re_ReferenceRecord = re.compile("ReferenceRecord\((\w{4}), '(.*?)'\);")
+    rr = re_ReferenceRecord.findall(data)
+    
+    records = [Record(r[0], r[1], d.flags) for r in rr]
+    return records
+
+
+def find_wbRefRecords(data):
+    records = find_concrete_wbRefRecords(data)
+    records = find_ReferenceRecords(data)
+    return records
+
+
+fdir = os.path.dirname(__file__)
+with open(os.path.join(fdir, "wbDefinitionsTES5.pas"), "r") as f:
+    inp = f.read()
+
+records = find_wbRecords(inp) + [find_stat(inp)] + find_wbRefRecords(inp)
+records.sort(key = lambda x: x.fourcc)
+print(len(records))
+
+class Encoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Jsonify):
+            return obj.encode()
+        else:
+            return json.JSONEncoder.default(self, obj)
+
+with open(os.path.join(fdir, "flags.json"), "w") as f:
+    json.dump(records, f, cls=Encoder, indent=4)

reference/record_filter.py

@@ -0,0 +1,95 @@
+import os
+import re
+import json
+from dataclasses import dataclass
+
+# helper used when dumping the records to JSON
+class Jsonify:
+    def encode(self, value=None):
+        if not value:
+            value = self.__dict__
+        return value
+
+# Record model
+@dataclass
+class Flag(Jsonify):
+    bit: int
+    description: str
+
+@dataclass
+class Record(Jsonify):
+    fourcc: str
+    name: str
+    flags: list[Flag]
+
+
+# open and read the records from xEdit source
+fdir = os.path.dirname(__file__)
+with open(os.path.join(fdir, "wbDefinitionsTES5.pas"), "r") as f:
+    inp = f.read()
+
+# find all wbRecord/wbRefRecord calls
+re_record = re.compile("(?:wbRecord|wbRefRecord)\((.*?\n(?:.*?\n)*?.*?)\);")    
+all = re_record.findall(inp)
+
+# find all ReferenceRecord calls
+re_ReferenceRecord = re.compile("ReferenceRecord\((\w{4}), '(.*?)'\);")
+rr = re_ReferenceRecord.findall(inp)
+
+# partition all into true records and the ReferenceRecord template
+[ref_template] = [x for x in all if x.startswith('aSignature')]
+all = [x for x in all if not x.startswith('aSignature')]
+
+# generate and add ReferenceRecords to all
+for sig, name in rr:
+    all += [ref_template.replace('aSignature', sig).replace('aName', "'" + name + "'")]
+    
+# parse strings into Records
+re_first = re.compile("(\w{4}), '(.*?)'")
+re_flags = re.compile(".*?\n.*?wbFlagsList\(.*?\n(?:.*?\n)+?.*?\]")
+re_flag_line = re.compile(".*?(\d+?),.*?'(.*)'")
+re_stat_flag = re.compile(".*?'(.*?)'")
+records = []
+for string in all:
+    # git sig and name from first line
+    sig, name = re_first.match(string).groups()
+    # check for a flag list, process if found
+    flags_string = re_flags.match(string)
+    flags = []
+    if flags_string:
+        flag_lines = flags_string.group(0).split('\n')[2:-1]
+        assert len(flag_lines) > 0
+        for line in flag_lines:
+            bit, desc = re_flag_line.match(line).groups()
+            flags += [Flag(bit, desc)]
+    # handle STAT special case
+    elif sig == "STAT":
+        flags_string = re.match(".*?\n.*?wbFlags\(.*?\n(?:.*?\n){32}.*?\]", string).group(0)
+        lines = flags_string.split('\n')[2:-1]
+        for bit, line in enumerate(lines):
+            desc = re_stat_flag.match(line).group(1)
+            if desc:
+                flags += [Flag(bit, desc)]
+    records += [Record(sig, name, flags)]
+    
+# remove duplicates
+rdict = {r.fourcc: r for r in records}
+records = list(rdict.values())
+
+# add in the apparently supported NOTE
+records += [Record("NOTE", "Note", [])]
+
+# sort the records by signature
+records = sorted(records, key=lambda x: x.fourcc)
+
+# write the records out to a JSON file
+class Encoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Jsonify):
+            return obj.encode()
+        else:
+            return json.JSONEncoder.default(self, obj)
+
+with open(os.path.join(fdir, "flags.json"), "w") as f:
+    json.dump(records, f, cls=Encoder, indent=4)
+    

reference/record_types.csv

@@ -0,0 +1,137 @@
+AACT,Action
+ACHR,Placed NPC
+ACTI,Activator
+ADDN,Addon Node
+ALCH,Ingestible
+AMMO,Ammunition
+ANIO,Animated Object
+APPA,Alchemical Apparatus
+ARMA,Armor Addon
+ARMO,Armor
+ARTO,Art Object
+ASPC,Acoustic Space
+ASTP,Association Type
+AVIF,Actor Value Information
+BOOK,Book
+BPTD,Body Part Data
+CAMS,Camera Shot
+CELL,Cell
+CLAS,Class
+CLDC,CLDC
+CLFM,Color
+CLMT,Climate
+COBJ,Constructible Object
+COLL,Collision Layer
+CONT,Container
+CPTH,Camera Path
+CSTY,Combat Style
+DEBR,Debris
+DIAL,Dialog Topic
+DLBR,Dialog Branch
+DLVW,Dialog View
+DOBJ,Default Object Manager
+DOOR,Door
+DUAL,Dual Cast Data
+ECZN,Encounter Zone
+EFSH,Effect Shader
+ENCH,Object Effect
+EQUP,Equip Type
+EXPL,Explosion
+EYES,Eyes
+FACT,Faction
+FLOR,Flora
+FLST,FormID List
+FSTP,Footstep
+FSTS,Footstep Set
+FURN,Furniture
+GLOB,Global
+GMST,Game Setting
+GRAS,Grass
+GRUP,Internal File Structure Group
+HAIR,HAIR
+HAZD,Hazard
+HDPT,Head Part
+IDLE,Idle Animation
+IDLM,Idle Marker
+IMAD,Image Space Adapter
+IMGS,Image Space
+INFO,Dialog response
+INGR,Ingredient
+IPCT,Impact
+IPDS,Impact Data Set
+KEYM,Key
+KYWD,Keyword
+LAND,Landscape
+LCRT,Location Reference Type
+LCTN,Location
+LENS,Lens Flare
+LGTM,Lighting Template
+LIGH,Light
+LSCR,Load Screen
+LTEX,Landscape Texture
+LVLI,Leveled Item
+LVLN,Leveled NPC
+LVSP,Leveled Spell
+MATO,Material Object
+MATT,Material Type
+MESG,Message
+MGEF,Magic Effect
+MISC,Misc. Item
+MOVT,Movement Type
+MSTT,Moveable Static
+MUSC,Music Type
+MUST,Music Track
+NAVI,Navigation Mesh Info Map
+NAVM,Navigation Mesh
+NOTE,NOTE
+NPC_,Non-Player Character (Actor)
+OTFT,Outfit
+PACK,Package
+PARW,Placed Arrow
+PBAR,Placed Barrier
+PBEA,Placed Beam
+PCON,Placed Cone/Voice
+PERK,Perk
+PFLA,Placed Flame
+PGRE,Placed Projectile
+PHZD,Placed Hazard
+PLYR,Player Reference
+PMIS,Placed Missile
+PROJ,Projectile
+PWAT,PWAT
+QUST,Quest
+RACE,Race
+REFR,Placed Object
+REGN,Region
+RELA,Relationship
+REVB,Reverb Parameters
+RFCT,Visual Effect
+RGDL,RGDL
+SCEN,Scene
+SCOL,SCOL
+SCOL,Static Collection
+SCPT,SCPT
+SCRL,Scroll
+SHOU,Shout
+SLGM,Soul Gem
+SMBN,Story Manager Branch Node
+SMEN,Story Manager Event Node
+SMQN,Story Manager Quest Node
+SNCT,Sound Category
+SNDR,Sound Descriptor
+SOPM,Sound Output Model
+SOUN,Sound Marker
+SPEL,Spell
+SPGD,Shader Particle Geometry
+STAT,Static
+TACT,Talking Activator
+TES4,Main File Header
+TREE,Tree
+TXST,Texture Set
+VOLI,Volumetric Lighting
+VTYP,Voice Type
+WATR,Water
+WEAP,Weapon
+WOOP,Word of Power
+WRLD,Worldspace
+WTHR,Weather