wmpmiles/Navmesher_old
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split out sized buffer type and added tests. Modified reader to better utilise sized buffer. Debugged and working at least as well as previous.

Browse Source
This commit is contained in:
William Miles
2022-09-09 14:59:56 +10:00
parent 392e226013
commit fa7025f774
10 changed files with 539 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
espReader/ESPReader.h
View File

@@ -24,10 +24,13 @@
*/
#include <stdint.h>
#include <string.h>
#undef NDEBUG
#include <assert.h>
#define NDEBUG
#include "msh.h"
#include "SizedBuffer.h"
// Guards for C++ usage
#ifdef __cplusplus
@@ -76,6 +79,7 @@ extern "C" {
typedef struct field Field;
typedef struct meta_node MetaNode;
typedef struct meta_tree MetaTree;
typedef struct esp_stats ESPStats;
//
@@ -113,11 +117,6 @@ extern "C" {
char _pad[4];
};
struct sized_buf {
char *data;
size_t size;
};
//
// === ENUMS ===
//
@@ -334,13 +333,6 @@ extern "C" {
return rth2rt[uint32_t_msh(type, RT_HASH_BITS, RT_HASH_SEED)];
}
// Updates a sized_buf after using `size` bytes
inline void sized_buf_update(struct sized_buf *sb, size_t size) {
assert(sb->size >= size);
sb->data += size;
sb->size -= size;
}
/* `espr_walk` walks through the tree structure of the esp/esm binary
* data starting at `data` of `size` bytes.
*
@@ -351,7 +343,7 @@ extern "C" {
* increasing in terms of memory location within the buffer.
*/
void espr_walk(
struct sized_buf esp,
SizedBuf esp,
struct walker_callbacks cb,
void *from_parent
);
@@ -359,13 +351,13 @@ extern "C" {
/* `espr_print` prints the header of every group and record in the given
* esp/esm binary data.
*/
void espr_print(struct sized_buf esp);
void espr_print(SizedBuf esp);
/* Calculates the number of groups and records in the esp/esm file and
* the size of the esp/esm if all of the compressed records were
* decompressed.
*/
struct esp_stats espr_stats(struct sized_buf esp);
struct esp_stats espr_stats(SizedBuf esp);
// Calculates the number of formid's in an esm/esp from the stats
inline uint32_t espr_formid_count(struct esp_stats stats) {
@@ -387,10 +379,7 @@ extern "C" {
* as it does so. buf_size should be the value returned from
* `espr_decompressed_size`, and `buf` should be at least of that size.
*/
void espr_decompress(
struct sized_buf esp,
struct sized_buf decom
);
void espr_decompress(SizedBuf esp, SizedBuf decom);
/* Constructs a MetaNode tree in `tree` over the esp/esm data in `in`.
*
@@ -398,13 +387,13 @@ extern "C" {
* data, and also allow for modifications that add, remove, or change
* the size of groups/records/fields.
*/
MetaTree espr_create_tree(struct sized_buf in, struct sized_buf tree);
MetaTree espr_create_tree(SizedBuf in, SizedBuf tree);
void espr_meta_walk(MetaTree tree, struct meta_callbacks cb);
void espr_meta_node_walk(MetaNode *m, struct meta_callbacks cb);
void espr_serialize(MetaTree tree, struct sized_buf out);
void espr_serialize(MetaTree tree, SizedBuf out);
// End C++ guard
#ifdef __cplusplus

207
espReader/Reader.c
View File

@@ -38,13 +38,9 @@ const int year_offset = 9;
void asserts(void);
// Tree walkers
char *walk_concat(
struct sized_buf tree,
struct walker_callbacks cb,
void *from_parent
);
char *walk_group(char *data, struct walker_callbacks cb, void *from_parent);
char *walk_record(char *data, struct walker_callbacks cb, void *from_parent);
void walk_concat(SizedBuf *tree, struct walker_callbacks cb, void *from_parent);
void walk_group(SizedBuf *tree, struct walker_callbacks cb, void *from_parent);
void walk_record(SizedBuf *tree, struct walker_callbacks cb, void *from_parent);
// Header printers
void print_group_header(Group *header);
@@ -118,21 +114,23 @@ void asserts(void)
}
void espr_walk(
struct sized_buf esp,
SizedBuf esp,
struct walker_callbacks cb,
void *from_parent
) {
// check assertions that cannot be checked at compile time
asserts();
char *data_start = esp.data;
// check that we are at the start of the file
const Type4 type = *(const Type4 *)esp.data;
assert(type.uint == rt[TES4]);
{
Type4 *type = NULL;
int err = sb_peek(esp, sizeof(Type4), &type);
assert(err == 0);
assert(type->uint == rt[TES4]);
}
esp.data = walk_concat(esp, cb, from_parent);
assert(esp.data == data_start + esp.size);
walk_concat(&esp, cb, from_parent);
assert(sb_empty(esp));
}
/* Unknown data will be some concatenation of groups and records.
@@ -140,27 +138,22 @@ void espr_walk(
* `walk_concat` will call the appropriate walking function
* for each segment of unknown data in this concatenation.
*/
char *walk_concat(
struct sized_buf tree,
struct walker_callbacks cb,
void *from_parent
) {
const char *end = tree.data + tree.size;
while (tree.data != end) {
assert(tree.data < end);
const Type4 *type = (Type4 *)tree.data;
void walk_concat(SizedBuf *tree, struct walker_callbacks cb, void *from_parent)
{
while (!sb_empty(*tree)) {
const Type4 *type = NULL;
int err = sb_peek(*tree, sizeof(Type4), &type);
assert(err == 0);
// check valid type
assert(rt[rt_hash(type->uint)] == type->uint);
// only need to distinguish between groups and records
if (type->uint == rt[GRUP])
tree.data = walk_group(tree.data, cb, from_parent);
walk_group(tree, cb, from_parent);
else
tree.data = walk_record(tree.data, cb, from_parent);
walk_record(tree, cb, from_parent);
}
return tree.data;
}
/* Walk a group record. Group records are containers for any other type of
@@ -169,18 +162,25 @@ char *walk_concat(
* This function will also call `cb` with the node constructed from this group
* record.
*/
char *walk_group(char *data, struct walker_callbacks cb, void *from_parent)
void walk_group(SizedBuf *tree, struct walker_callbacks cb, void *from_parent)
{
Group *const header = (Group *const)data;
Group *header = NULL;
{
int err = sb_peek(*tree, sizeof(Group), &header);
assert(err == 0);
}
// The size in the group header includes the size of the header
char *data_start = data + sizeof(Group);
char *data_end = data + header->size;
size_t data_size = data_end - data_start;
SizedBuf group = { .size = header->size };
{
int err = sb_recast(tree, group.size, &group.data);
assert(err == 0);
err = sb_recast(&group, sizeof(Group), &header);
assert(err == 0);
}
Node n = {
.header.group = header,
.data = data_start,
.data = group.data,
.type = NT_GROUP
};
void *carry = NULL;
@@ -191,27 +191,30 @@ char *walk_group(char *data, struct walker_callbacks cb, void *from_parent)
cb.pre(n, cb.data, &carry, from_parent, &to_children);
// Walk through the concatenation of data inside the group.
struct sized_buf tree = { data_start, data_size };
data = walk_concat(tree, cb, to_children);
assert(data == data_end);
walk_concat(&group, cb, to_children);
assert(sb_empty(group));
// Post-walk callback
if (cb.post)
cb.post(n, cb.data, carry);
return data;
}
char *walk_record(char *data, struct walker_callbacks cb, void *from_parent)
void walk_record(SizedBuf *tree, struct walker_callbacks cb, void *from_parent)
{
Record *header = (Record *)data;
assert(header->type.uint != rt[GRUP]);
Record *header = NULL;
char *data = NULL;
{
int err = sb_recast(tree, sizeof(Record), &header);
assert(err == 0);
err = sb_recast(tree, header->size, &data);
assert(err == 0);
}
char *data_start = data + sizeof(Record);
assert(header->type.uint != rt[GRUP]);
Node n = {
.header.record = header,
.data = data_start,
.data = data,
.type = NT_RECORD
};
void *carry = NULL;
@@ -226,17 +229,12 @@ char *walk_record(char *data, struct walker_callbacks cb, void *from_parent)
if (cb.pre)
cb.pre(n, cb.data, &carry, from_parent, &to_children);
// Update data ptr based on record size.
data += sizeof(Record) + header->size;
// Post-walk callback
if (cb.post)
cb.post(n, cb.data, carry);
return data;
}
void espr_print(struct sized_buf esp)
void espr_print(SizedBuf esp)
{
struct walker_callbacks cb = { .pre = print_cb };
espr_walk(esp, cb, NULL);
@@ -265,9 +263,9 @@ void print_cb(
}
}
struct esp_stats espr_stats(struct sized_buf esp)
ESPStats espr_stats(SizedBuf esp)
{
struct esp_stats stats = { 0 };
ESPStats stats = { 0 };
struct walker_callbacks cb = { .pre = stats_cb, .data = &stats };
espr_walk(esp, cb, NULL);
return stats;
@@ -310,7 +308,7 @@ void stats_cb(
}
}
void espr_decompress(struct sized_buf esp, struct sized_buf decom)
void espr_decompress(SizedBuf esp, SizedBuf decom)
{
struct walker_callbacks cb = {
.pre = decompress_pre,
@@ -334,7 +332,7 @@ void espr_decompress(struct sized_buf esp, struct sized_buf decom)
*/
void decompress_pre(
Node n,
void *decom_ptr,
void *out_buf,
void **carry_out,
void *from_parent,
void **to_children
@@ -342,28 +340,32 @@ void decompress_pre(
(void)from_parent;
(void)to_children;
struct sized_buf *d = decom_ptr;
SizedBuf *d = out_buf;
switch (n.type) {
case NT_RECORD:
// compressed record
if (n.header.record->flags & COMPRESSED_FLAG) {
// copy header
memcpy(d->data, n.header.record, sizeof(Record));
// copied header reference
Record *header = (Record *)d->data;
// update decom struct
sized_buf_update(d, sizeof(Record));
Record *copied_header = NULL;
{
// Acces for copied header
int err = sb_peek(*d, sizeof(Record), &copied_header);
assert(err == 0);
// copy header
err = sb_copyin(d, (char *)n.header.record, sizeof(Record));
assert(err == 0);
}
// decompress directly into buffer
// first 4 bytes are the decompressed size
const uint32_t dc_size = *((uint32_t *)n.data);
uint32_t to_copy = dc_size;
uint32_t cur_size =
n.header.record->size - sizeof(uint32_t);
uint32_t cur_size = n.header.record->size - sizeof(uint32_t);
char *data_start = n.data + sizeof(uint32_t);
// check that we have enough space in the buffer
sb_check(*d, dc_size);
int ret = uncompress(
(Bytef *)d->data,
(uLongf *)&to_copy,
@@ -373,32 +375,27 @@ void decompress_pre(
assert(ret == Z_OK);
assert(to_copy == dc_size);
// update decom struct
sized_buf_update(d, dc_size);
// update the buffer
sb_update(d, dc_size);
// update header data size
header->size = dc_size;
copied_header->size = dc_size;
// unset compressed flag
header->flags &= ~COMPRESSED_FLAG;
copied_header->flags &= ~COMPRESSED_FLAG;
} else {
// copy record
size_t record_size = sizeof(Record) + n.header.record->size;
memcpy(d->data, n.header.record, record_size);
// update decom
sized_buf_update(d, record_size);
size_t record_size = sizeof(Record)
+ n.header.record->size;
sb_copyin(d, (char *)n.header.record, record_size);
}
break;
case NT_GROUP:
// copy header, contents will be copied while walking
memcpy(d->data, n.header.group, sizeof(Group));
// save copied header location for post-walk group size recalc
*carry_out = (void *)d->data;
// update decom
sized_buf_update(d, sizeof(Group));
// copy header, contents will be copied while walking
sb_copyin(d, (char *)n.header.group, sizeof(Group));
break;
default:
@@ -412,9 +409,9 @@ void decompress_pre(
* based on the difference between the current destination pointer and the
* group header pointer.
*/
void decompress_post(Node n, void *decom_ptr, void *carry_in)
void decompress_post(Node n, void *out, void *carry_in)
{
struct sized_buf *d = decom_ptr;
SizedBuf *d = out;
// only need to handle group resize
if (n.type == NT_GROUP) {
@@ -630,15 +627,16 @@ Timestamp convert_ts(uint16_t ts)
return (Timestamp) { year, month, day };
}
MetaTree espr_create_tree(struct sized_buf in, struct sized_buf tree)
MetaTree espr_create_tree(SizedBuf in, SizedBuf tree)
{
// create root node
MetaNode *root = (MetaNode *)tree.data;
MetaNode *root = NULL;
{
int err = sb_recast(&tree, sizeof(MetaNode), &root);
assert(err == 0);
}
*root = (MetaNode){ 0 };
// update tree
sized_buf_update(&tree, sizeof(MetaNode));
// walk
struct walker_callbacks cb = { .pre = create_tree_cb, .data = &tree };
espr_walk(in, cb, root);
@@ -656,9 +654,12 @@ void create_tree_cb(
(void)carry_out;
// add new metanode to tree
struct sized_buf *tree = data;
MetaNode *m = (MetaNode *)tree->data;
sized_buf_update(tree, sizeof(MetaNode));
SizedBuf *tree = data;
MetaNode *m = NULL;
{
int err = sb_recast(tree, sizeof(MetaNode), &m);
assert(err == 0);
}
// parent passes their MetaNode to children
MetaNode *p = from_parent;
@@ -692,13 +693,13 @@ void espr_meta_node_walk(MetaNode *m, struct meta_callbacks cb) {
}
}
void espr_serialize(MetaTree tree, struct sized_buf out) {
void espr_serialize(MetaTree tree, SizedBuf out) {
struct meta_callbacks cb = { .pre = serialize_cb, .data = &out };
espr_meta_walk(tree, cb);
}
void serialize_cb(MetaNode *m, void *data) {
struct sized_buf *out = data;
SizedBuf *out = data;
// exit on empty node
if (!m->n.data)
@@ -707,19 +708,25 @@ void serialize_cb(MetaNode *m, void *data) {
switch (m->n.type) {
case NT_GROUP:
// only serialize the header of groups
assert(out->size >= sizeof(Group));
memcpy(out->data, m->n.header.group, sizeof(Group));
sized_buf_update(out, sizeof(Group));
break;
{
int err = sb_copyin(out, (char *)m->n.header.group, sizeof(Group));
assert(err == 0);
}
break;
case NT_RECORD:
size_t data_size = m->n.header.record->size;
assert(out->size >= data_size + sizeof(Record));
char *header = (char *)m->n.header.record;
// serialize header and data separately as they may be
// discontiguous
memcpy(out->data, m->n.header.record, sizeof(Record));
sized_buf_update(out, sizeof(Record));
memcpy(out->data, m->n.data, data_size);
sized_buf_update(out, data_size);
{
int err = sb_copyin(out, header, sizeof(Record));
assert(err == 0);
err = sb_copyin(out, m->n.data, data_size);
assert(err == 0);
}
break;
default:
assert(false);
}
}

7
espReader/espReader.vcxproj
View File

@@ -111,7 +111,7 @@
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpp17</LanguageStandard>
<LanguageStandard_C>stdc11</LanguageStandard_C>
<AdditionalIncludeDirectories>..\zlib-win-build</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>..\zlib-win-build;..\SizedBuffer</AdditionalIncludeDirectories>
<Optimization>Disabled</Optimization>
</ClCompile>
<Link>
@@ -129,7 +129,7 @@
<PreprocessorDefinitions>NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard_C>stdc11</LanguageStandard_C>
<AdditionalIncludeDirectories>..\zlib-win-build</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>..\zlib-win-build;..\SizedBuffer</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<SubSystem>
@@ -148,6 +148,9 @@
<ClCompile Include="Reader.c" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\SizedBuffer\SizedBuffer.vcxproj">
<Project>{cb1c8f66-5b90-4de7-890b-f6430daaf25f}</Project>
</ProjectReference>
<ProjectReference Include="..\zlib-win-build\build-VS2022\libz-static\libz-static.vcxproj">
<Project>{b56d17bc-072b-42f3-844a-870a07afbaaa}</Project>
</ProjectReference>