From 392e226013ee677cc8f1db50905f100436d3255d Mon Sep 17 00:00:00 2001 From: William Miles Date: Thu, 8 Sep 2022 19:35:16 +1000 Subject: [PATCH] Still haven't properly tested MetaTree. But have all of the current features working together. --- NavmeshList/NavmeshList.vcxproj | 1 + NavmeshList/main.c | 113 ++++++++++++++++++------- espReader/ESPReader.h | 32 ++++++-- espReader/Reader.c | 141 ++++++++++++++++++++------------ espReader/espReader.vcxproj | 2 +- 5 files changed, 195 insertions(+), 94 deletions(-) diff --git a/NavmeshList/NavmeshList.vcxproj b/NavmeshList/NavmeshList.vcxproj index 5e70355..b79e729 100644 --- a/NavmeshList/NavmeshList.vcxproj +++ b/NavmeshList/NavmeshList.vcxproj @@ -72,6 +72,7 @@ {4ce7e7fd-7fff-4c42-ab16-c9c2b01e5d56} NavmeshList 10.0 + NavmeshList diff --git a/NavmeshList/main.c b/NavmeshList/main.c index 8cd0b35..2d28224 100644 --- a/NavmeshList/main.c +++ b/NavmeshList/main.c @@ -11,50 +11,103 @@ #include int main(void) { - FILE *fp; - errno_t ret = fopen_s(&fp, "Skyrim.esm", "rb"); + errno_t ret = 0; - if (ret || !fp) - return ret; + // Read in the esp file + struct sized_buf esp_buf = { 0 }; + { + FILE *fp; + ret = fopen_s(&fp, "Skyrim.esm", "rb"); - fseek(fp, 0L, SEEK_END); - size_t size = ftell(fp); - rewind(fp); + if (ret || !fp) + goto exit1; - char *buffer = malloc(size); - if (!buffer) - return errno; + fseek(fp, 0L, SEEK_END); + esp_buf.size = ftell(fp); + rewind(fp); - size_t read = fread(buffer, sizeof(char), size, fp); - assert(read == size); + esp_buf.data = malloc(esp_buf.size); + if (!esp_buf.data) { + ret = errno; + goto exit1; + } - struct sized_buf esp = { .data = buffer, .size = size }; + size_t read = fread(esp_buf.data, sizeof(char), esp_buf.size, fp); + fclose(fp); + assert(read == esp_buf.size); + } - struct esp_stats stats = espr_stats(buffer, size); + // Calculate esp stats + struct esp_stats stats = espr_stats(esp_buf); - /* - char *decompressed = malloc(stats.decompressed_size); - if (!decompressed) - return errno; + // Decompress the esp file + struct sized_buf decom_buf = { 0 }; + { + decom_buf.size = stats.decompressed_size; + decom_buf.data = malloc(decom_buf.size); + if (!decom_buf.data) { + ret = errno; + goto exit2; + } - // espr_print(buffer, size); + espr_decompress(esp_buf, decom_buf); + } - espr_decompress(buffer, size, decompressed, stats.decompressed_size); - */ + // Construct the meta tree + struct sized_buf tree_buf = { 0 }; + MetaTree tree = { 0 }; + { + tree_buf.size = espr_tree_size(stats); + tree_buf.data = malloc(tree_buf.size); + if (!tree_buf.data) { + ret = errno; + goto exit3; + } + tree = espr_create_tree(decom_buf, tree_buf); + } - size_t tree_size = espr_tree_size(stats); - char *tree_data = malloc(tree_size); - struct sized_buf tree = { .data = tree_data, .size = tree_size }; + // Serialize and write out the uncompressed esp + struct sized_buf serialized_buf = { 0 }; + { + serialized_buf.size = tree.size; + serialized_buf.data = malloc(serialized_buf.size); + if (!serialized_buf.data) { + ret = errno; + goto exit4; + } - MetaNode *root = espr_create_tree(esp, tree); + espr_serialize(tree, serialized_buf); - free(buffer); + printf("Serialized.\n"); - /* - espr_print(decompressed, stats.decompressed_size); + FILE *fp; + ret = fopen_s(&fp, "Test.esm", "wb"); - free(decompressed); - */ + if (ret || !fp) + goto exit5; + + size_t written = fwrite(serialized_buf.data, sizeof(char), + serialized_buf.size, fp); + fclose(fp); + assert(written == serialized_buf.size); + + printf("Written.\n"); + } + +exit5: + free(serialized_buf.data); +exit4: + free(tree_buf.data); +exit3: + free(decom_buf.data); +exit2: + free(esp_buf.data); +exit1: + // lazy file pointer cleanup + _fcloseall(); + + printf("Finished.\n"); + (void)getc(stdin); return 0; } diff --git a/espReader/ESPReader.h b/espReader/ESPReader.h index 1f58b63..dcdecca 100644 --- a/espReader/ESPReader.h +++ b/espReader/ESPReader.h @@ -75,6 +75,7 @@ extern "C" { typedef struct record Record; typedef struct field Field; typedef struct meta_node MetaNode; + typedef struct meta_tree MetaTree; // @@ -215,6 +216,11 @@ extern "C" { void *data; }; + struct meta_callbacks { + void (*pre)(MetaNode *m, void *data); + void *data; + }; + /* Meta Nodes are used for constructing a more flexible tree structure * on top of the natural structure of ESP/ESM files. * @@ -249,6 +255,11 @@ extern "C" { MetaNode *next; }; + struct meta_tree { + MetaNode *root; + size_t size; + }; + // // === BINARY DATA OVERLAYS === // @@ -340,8 +351,7 @@ extern "C" { * increasing in terms of memory location within the buffer. */ void espr_walk( - char *data, - size_t size, + struct sized_buf esp, struct walker_callbacks cb, void *from_parent ); @@ -349,13 +359,13 @@ extern "C" { /* `espr_print` prints the header of every group and record in the given * esp/esm binary data. */ - void espr_print(char *data, size_t size); + void espr_print(struct sized_buf esp); /* Calculates the number of groups and records in the esp/esm file and * the size of the esp/esm if all of the compressed records were * decompressed. */ - struct esp_stats espr_stats(char *data, size_t size); + struct esp_stats espr_stats(struct sized_buf esp); // Calculates the number of formid's in an esm/esp from the stats inline uint32_t espr_formid_count(struct esp_stats stats) { @@ -378,10 +388,8 @@ extern "C" { * `espr_decompressed_size`, and `buf` should be at least of that size. */ void espr_decompress( - char *data, - size_t size, - char *buf, - size_t buf_size + struct sized_buf esp, + struct sized_buf decom ); /* Constructs a MetaNode tree in `tree` over the esp/esm data in `in`. @@ -390,7 +398,13 @@ extern "C" { * data, and also allow for modifications that add, remove, or change * the size of groups/records/fields. */ - MetaNode *espr_create_tree(struct sized_buf in, struct sized_buf tree); + MetaTree espr_create_tree(struct sized_buf in, struct sized_buf tree); + + void espr_meta_walk(MetaTree tree, struct meta_callbacks cb); + + void espr_meta_node_walk(MetaNode *m, struct meta_callbacks cb); + + void espr_serialize(MetaTree tree, struct sized_buf out); // End C++ guard #ifdef __cplusplus diff --git a/espReader/Reader.c b/espReader/Reader.c index 016debc..4298f1f 100644 --- a/espReader/Reader.c +++ b/espReader/Reader.c @@ -39,8 +39,7 @@ void asserts(void); // Tree walkers char *walk_concat( - char *data, - size_t size, + struct sized_buf tree, struct walker_callbacks cb, void *from_parent ); @@ -88,7 +87,7 @@ void decompress_pre( void decompress_post( Node n, void *data, - void **carry_in + void *carry_in ); void create_tree_cb( Node n, @@ -97,6 +96,7 @@ void create_tree_cb( void *from_parent, void **to_children ); +void serialize_cb(MetaNode *m, void *data); // // === FUNCTIONS === @@ -118,22 +118,21 @@ void asserts(void) } void espr_walk( - char *data, - size_t size, + struct sized_buf esp, struct walker_callbacks cb, void *from_parent ) { // check assertions that cannot be checked at compile time asserts(); - char *data_start = data; + char *data_start = esp.data; // check that we are at the start of the file - const Type4 type = *(const Type4 *)data; + const Type4 type = *(const Type4 *)esp.data; assert(type.uint == rt[TES4]); - data = walk_concat(data, size, cb, from_parent); - assert(data == data_start + size); + esp.data = walk_concat(esp, cb, from_parent); + assert(esp.data == data_start + esp.size); } /* Unknown data will be some concatenation of groups and records. @@ -142,27 +141,26 @@ void espr_walk( * for each segment of unknown data in this concatenation. */ char *walk_concat( - char *data, - size_t size, + struct sized_buf tree, struct walker_callbacks cb, void *from_parent ) { - const char *end = data + size; - while (data != end) { - assert(data < end); + const char *end = tree.data + tree.size; + while (tree.data != end) { + assert(tree.data < end); - const Type4 *type = (Type4 *)data; + const Type4 *type = (Type4 *)tree.data; // check valid type assert(rt[rt_hash(type->uint)] == type->uint); // only need to distinguish between groups and records if (type->uint == rt[GRUP]) - data = walk_group(data, cb, from_parent); + tree.data = walk_group(tree.data, cb, from_parent); else - data = walk_record(data, cb, from_parent); + tree.data = walk_record(tree.data, cb, from_parent); } - return data; + return tree.data; } /* Walk a group record. Group records are containers for any other type of @@ -193,7 +191,8 @@ char *walk_group(char *data, struct walker_callbacks cb, void *from_parent) cb.pre(n, cb.data, &carry, from_parent, &to_children); // Walk through the concatenation of data inside the group. - data = walk_concat(data_start, data_size, cb, to_children); + struct sized_buf tree = { data_start, data_size }; + data = walk_concat(tree, cb, to_children); assert(data == data_end); // Post-walk callback @@ -237,10 +236,10 @@ char *walk_record(char *data, struct walker_callbacks cb, void *from_parent) return data; } -void espr_print(char *data, size_t size) +void espr_print(struct sized_buf esp) { struct walker_callbacks cb = { .pre = print_cb }; - espr_walk(data, size, cb, NULL); + espr_walk(esp, cb, NULL); } void print_cb( @@ -266,11 +265,11 @@ void print_cb( } } -struct esp_stats espr_stats(char *data, size_t size) +struct esp_stats espr_stats(struct sized_buf esp) { struct esp_stats stats = { 0 }; struct walker_callbacks cb = { .pre = stats_cb, .data = &stats }; - espr_walk(data, size, cb, NULL); + espr_walk(esp, cb, NULL); return stats; } @@ -311,20 +310,14 @@ void stats_cb( } } -struct decom { - char *buf; - size_t remaining; -}; - -void espr_decompress(char *data, size_t size, char *buf, size_t buf_size) +void espr_decompress(struct sized_buf esp, struct sized_buf decom) { - struct decom s = { .buf = buf, .remaining = buf_size }; struct walker_callbacks cb = { .pre = decompress_pre, .post = decompress_post, - .data = &s + .data = &decom }; - espr_walk(data, size, cb, NULL); + espr_walk(esp, cb, NULL); } /* Handles the copying of groups and records, and the decompression of @@ -349,21 +342,20 @@ void decompress_pre( (void)from_parent; (void)to_children; - struct decom *d = decom_ptr; + struct sized_buf *d = decom_ptr; switch (n.type) { case NT_RECORD: // compressed record if (n.header.record->flags & COMPRESSED_FLAG) { // copy header - memcpy(d->buf, n.header.record, sizeof(Record)); + memcpy(d->data, n.header.record, sizeof(Record)); // copied header reference - Record *header = (Record *)d->buf; + Record *header = (Record *)d->data; // update decom struct - d->remaining -= sizeof(Record); - d->buf += sizeof(Record); + sized_buf_update(d, sizeof(Record)); // decompress directly into buffer // first 4 bytes are the decompressed size @@ -373,7 +365,7 @@ void decompress_pre( n.header.record->size - sizeof(uint32_t); char *data_start = n.data + sizeof(uint32_t); int ret = uncompress( - (Bytef *)d->buf, + (Bytef *)d->data, (uLongf *)&to_copy, (Bytef *)data_start, (uLong)cur_size @@ -382,8 +374,7 @@ void decompress_pre( assert(to_copy == dc_size); // update decom struct - d->remaining -= dc_size; - d->buf += dc_size; + sized_buf_update(d, dc_size); // update header data size header->size = dc_size; @@ -393,23 +384,21 @@ void decompress_pre( } else { // copy record size_t record_size = sizeof(Record) + n.header.record->size; - memcpy(d->buf, n.header.record, record_size); + memcpy(d->data, n.header.record, record_size); // update decom - d->remaining -= record_size; - d->buf += record_size; + sized_buf_update(d, record_size); } break; case NT_GROUP: // copy header, contents will be copied while walking - memcpy(d->buf, n.header.group, sizeof(Group)); + memcpy(d->data, n.header.group, sizeof(Group)); // save copied header location for post-walk group size recalc - *carry_out = (void *)d->buf; + *carry_out = (void *)d->data; // update decom - d->buf += sizeof(Group); - d->remaining -= sizeof(Group); + sized_buf_update(d, sizeof(Group)); break; default: @@ -423,14 +412,14 @@ void decompress_pre( * based on the difference between the current destination pointer and the * group header pointer. */ -void decompress_post(Node n, void *decom_ptr, void **carry_in) +void decompress_post(Node n, void *decom_ptr, void *carry_in) { - struct decom *d = decom_ptr; + struct sized_buf *d = decom_ptr; // only need to handle group resize if (n.type == NT_GROUP) { - Group *g = (Group *)(*carry_in); - uint32_t new_size = (uint32_t)((char *)d->buf - (char *)g); + Group *g = (Group *)carry_in; + uint32_t new_size = (uint32_t)((char *)d->data - (char *)g); g->size = new_size; } } @@ -641,7 +630,7 @@ Timestamp convert_ts(uint16_t ts) return (Timestamp) { year, month, day }; } -MetaNode *espr_create_tree(struct sized_buf in, struct sized_buf tree) +MetaTree espr_create_tree(struct sized_buf in, struct sized_buf tree) { // create root node MetaNode *root = (MetaNode *)tree.data; @@ -652,9 +641,9 @@ MetaNode *espr_create_tree(struct sized_buf in, struct sized_buf tree) // walk struct walker_callbacks cb = { .pre = create_tree_cb, .data = &tree }; - espr_walk(in.data, in.size, cb, root); + espr_walk(in, cb, root); - return root; + return (MetaTree) { .root = root, .size = in.size }; } void create_tree_cb( @@ -690,3 +679,47 @@ void create_tree_cb( *to_children = m; } +void espr_meta_walk(MetaTree tree, struct meta_callbacks cb) { + espr_meta_node_walk(tree.root, cb); +} + +void espr_meta_node_walk(MetaNode *m, struct meta_callbacks cb) { + cb.pre(m, cb.data); + MetaNode *child = m->first_child; + while (child) { + espr_meta_node_walk(child, cb); + child = child->next; + } +} + +void espr_serialize(MetaTree tree, struct sized_buf out) { + struct meta_callbacks cb = { .pre = serialize_cb, .data = &out }; + espr_meta_walk(tree, cb); +} + +void serialize_cb(MetaNode *m, void *data) { + struct sized_buf *out = data; + + // exit on empty node + if (!m->n.data) + return; + + switch (m->n.type) { + case NT_GROUP: + // only serialize the header of groups + assert(out->size >= sizeof(Group)); + memcpy(out->data, m->n.header.group, sizeof(Group)); + sized_buf_update(out, sizeof(Group)); + break; + case NT_RECORD: + size_t data_size = m->n.header.record->size; + assert(out->size >= data_size + sizeof(Record)); + // serialize header and data separately as they may be + // discontiguous + memcpy(out->data, m->n.header.record, sizeof(Record)); + sized_buf_update(out, sizeof(Record)); + memcpy(out->data, m->n.data, data_size); + sized_buf_update(out, data_size); + } +} + diff --git a/espReader/espReader.vcxproj b/espReader/espReader.vcxproj index 0e16c11..4f5460c 100644 --- a/espReader/espReader.vcxproj +++ b/espReader/espReader.vcxproj @@ -111,7 +111,7 @@ true stdcpp17 stdc11 - ..\zlib + ..\zlib-win-build Disabled