Files
MeanField_SERiF/src/lib/serif/utils/serialize/archive.cpp

425 lines
22 KiB
C++
Raw Normal View History

#include "serif/utils/serialize/archive.hpp"
#include <algorithm>
#include <atomic>
#include <charconv>
#include <chrono>
#include <cstdio>
#include <iomanip>
#include <limits>
#include <locale>
#include <set>
#include <sstream>
#include <system_error>
#include "serif/config.hpp"
#include "format.hpp"
namespace serif::utils::serialize {
namespace {
struct Header {
Format format{Format::binary};
std::string version;
std::string timestamp;
std::uint64_t index_offset{detail::header_bytes};
std::uint64_t index_bytes{0};
std::uint64_t data_offset{0};
std::uint64_t file_bytes{0};
std::uint64_t entries{0};
std::string checksum;
};
struct Payload {
Entry entry;
std::string bytes;
};
/** @brief Keep the fixed header readable and independent of compiler structure padding. */
std::string print_header(const Header& header) {
std::ostringstream text;
text.imbue(std::locale::classic());
text << "SERIF_ARCHIVE 1\nformat " << (header.format == Format::binary ? "binary" : "text")
<< "\nserif_version " << std::quoted(header.version) << "\nwritten_utc " << std::quoted(header.timestamp)
<< "\nledger_offset " << header.index_offset << "\nledger_bytes " << header.index_bytes
<< "\ndata_offset " << header.data_offset << "\nfile_bytes " << header.file_bytes
<< "\nentries " << header.entries << "\nledger_checksum " << header.checksum << "\nEND_HEADER\n";
std::string result = text.str();
result.resize(detail::header_bytes, ' ');
result.back() = '\n';
return result;
}
/** @brief Parse a fixed sequence so duplicated or unknown control fields cannot be ambiguous. */
Result<Header> parse_header(std::string_view bytes) {
std::istringstream input{std::string{bytes}};
input.imbue(std::locale::classic());
Header header;
std::string name;
std::string format;
int version = 0;
/** @brief Consume the next required control-field name. */
const auto next = [&](std::string_view expected) { return static_cast<bool>(input >> name) && name == expected; };
const bool valid = next("SERIF_ARCHIVE") && (input >> version) && version == 1 && next("format") && (input >> format)
&& next("serif_version") && (input >> std::quoted(header.version)) && next("written_utc") && (input >> std::quoted(header.timestamp))
&& next("ledger_offset") && (input >> header.index_offset) && next("ledger_bytes") && (input >> header.index_bytes)
&& next("data_offset") && (input >> header.data_offset) && next("file_bytes") && (input >> header.file_bytes)
&& next("entries") && (input >> header.entries) && next("ledger_checksum") && (input >> header.checksum) && next("END_HEADER");
input >> std::ws;
if (!valid || !input.eof() || (format != "binary" && format != "text")) {
return std::unexpected(Error{"header", "Malformed header or unsupported archive format version."});
}
header.format = format == "binary" ? Format::binary : Format::text;
return header;
}
/** @brief Compare semantic major versions without interpreting arbitrary strings as zero. */
std::optional<std::uint64_t> major_version(std::string_view version) {
const std::size_t dot = version.find('.');
if (dot == std::string_view::npos || dot == 0) {
return {};
}
std::uint64_t major = 0;
const auto parsed = std::from_chars(version.data(), version.data() + dot, major);
if (parsed.ec != std::errc{} || parsed.ptr != version.data() + dot) {
return {};
}
return major;
}
/** @brief Flatten a component tree into independent payloads and reject ambiguous names. */
Result<void> collect(const Node& node, std::string path, Format format, std::vector<Payload>& result, std::size_t depth = 0) {
if (!detail::valid_path(path) || depth > 128 || node.version == 0 || node.type.empty() || node.type.size() > 1024
|| node.type.find_first_of("\r\n\0", 0, 3) != std::string::npos) {
return std::unexpected(Error{std::move(path), "Invalid component path, schema identifier, or nesting depth."});
}
if (!node.children.empty() && !std::holds_alternative<std::monostate>(node.value)) {
return std::unexpected(Error{std::move(path), "A component must contain either fields or a payload."});
}
Payload payload{
.entry = {.path = path, .type = node.type, .version = node.version, .kind = static_cast<std::uint32_t>(node.value.index()),
.count = detail::value_count(node.value)},
.bytes = detail::encode_value(node.value, format)};
payload.entry.bytes = payload.bytes.size();
payload.entry.checksum = fingerprint(payload.bytes);
result.push_back(std::move(payload));
for (const auto& [name, child] : node.children) {
if (name.find('/') != std::string::npos) {
return std::unexpected(Error{path, "A field name cannot contain a path separator."});
}
auto saved = collect(child, path + "/" + name, format, result, depth + 1);
if (!saved) {
return saved;
}
}
return {};
}
/** @brief Fixed-width offsets keep the ledger length unchanged when final locations are filled in. */
std::string print_ledger(const std::vector<Payload>& payloads) {
std::ostringstream output;
output.imbue(std::locale::classic());
output << "# path schema version kind count offset bytes checksum\n";
for (const Payload& payload : payloads) {
const Entry& entry = payload.entry;
output << std::quoted(entry.path) << ' ' << std::quoted(entry.type) << ' ' << entry.version << ' ' << entry.kind << ' '
<< entry.count << ' ' << std::setfill('0') << std::setw(20) << entry.offset << ' ' << std::setw(20) << entry.bytes
<< ' ' << entry.checksum << '\n';
}
return output.str();
}
/** @brief Give each text payload a visible section label outside its indexed bytes. */
std::string section_label(const Entry& entry, Format format) {
return format == Format::text ? "\n# " + entry.path + "\n" : "";
}
/** @brief Remove a failed temporary write without ever removing the previous destination. */
struct TemporaryFile {
std::filesystem::path path;
std::FILE* stream{nullptr};
bool owned{false};
/** @brief Clean up regardless of which validation or I/O operation failed. */
~TemporaryFile() {
if (stream != nullptr) {
std::fclose(stream);
}
if (owned) {
std::error_code error;
std::filesystem::remove(path, error);
}
}
/** @brief Check every byte of an output section. */
bool append(std::string_view bytes) const { return std::fwrite(bytes.data(), 1, bytes.size(), stream) == bytes.size(); }
};
/** @brief Read an exact bounded region, clearing EOF from any preceding failed read. */
Result<std::string> read_bytes(std::ifstream& file, std::uint64_t offset, std::uint64_t bytes, std::string_view path) {
if (bytes > static_cast<std::uint64_t>(std::numeric_limits<std::streamsize>::max())
|| offset > static_cast<std::uint64_t>(std::numeric_limits<std::streamoff>::max())) {
return std::unexpected(Error{std::string{path}, "File region exceeds this reader's addressable range."});
}
file.clear();
file.seekg(static_cast<std::streamoff>(offset));
std::string result(bytes, '\0');
file.read(result.data(), static_cast<std::streamsize>(bytes));
if (!file) {
return std::unexpected(Error{std::string{path}, "Unable to read the complete indexed file region."});
}
return result;
}
/** @brief Validate all structural bounds before any payload is allocated or interpreted. */
Result<std::vector<Entry>> parse_ledger(std::string_view bytes, const Header& header, const ReadOptions& options) {
std::istringstream input{std::string{bytes}};
input.imbue(std::locale::classic());
std::string line;
std::getline(input, line);
if (line != "# path schema version kind count offset bytes checksum") {
return std::unexpected(Error{"ledger", "Unrecognized ledger columns."});
}
std::vector<Entry> entries;
std::map<std::string, std::uint32_t> kinds;
std::uint64_t previous_end = header.data_offset;
while (std::getline(input, line)) {
Entry entry;
std::istringstream row{line};
row.imbue(std::locale::classic());
const bool parsed = static_cast<bool>(row >> std::quoted(entry.path) >> std::quoted(entry.type) >> entry.version >> entry.kind
>> entry.count >> entry.offset >> entry.bytes >> entry.checksum);
row >> std::ws;
if (!parsed || !row.eof() || !detail::valid_path(entry.path) || entry.type.empty() || entry.type.size() > 1024
|| entry.kind > 8 || entry.version == 0 || entries.size() >= options.maximum_entries
|| static_cast<std::uint64_t>(std::count(entry.path.begin(), entry.path.end(), '/')) > options.maximum_depth
|| (!entries.empty() && entries.back().path >= entry.path)) {
return std::unexpected(Error{entry.path, "Malformed, duplicated, unsorted, or oversized ledger entry."});
}
const std::size_t separator = entry.path.rfind('/');
if (separator != std::string::npos) {
const auto parent = kinds.find(entry.path.substr(0, separator));
if (parent == kinds.end() || parent->second != 0) {
return std::unexpected(Error{entry.path, "A field's parent must be an existing component directory."});
}
}
if (entry.kind == 0) {
if (entry.count != 0 || entry.offset != 0 || entry.bytes != 0 || entry.checksum != fingerprint("")) {
return std::unexpected(Error{entry.path, "Directory entries cannot reference payload bytes."});
}
} else {
if (entry.offset < previous_end || entry.offset > header.file_bytes || entry.bytes > header.file_bytes - entry.offset
|| entry.bytes > options.maximum_payload_bytes || entry.count > options.maximum_payload_bytes) {
return std::unexpected(Error{entry.path, "Payload overlaps another section, exceeds the file, or exceeds configured limits."});
}
previous_end = entry.offset + entry.bytes;
}
kinds.emplace(entry.path, entry.kind);
entries.push_back(std::move(entry));
}
if (entries.size() != header.entries || !kinds.contains("provenance")) {
return std::unexpected(Error{"ledger", "Entry count is inconsistent or required provenance is missing."});
}
return entries;
}
}
void add_component(Pack& contents, Field component) {
if (contents.error) {
return;
}
if (component.name == "provenance" || !detail::valid_path(component.name) || component.name.find('/') != std::string::npos) {
contents.error = Error{component.name, "Use a non-reserved, single-component archive name."};
return;
}
const auto found = contents.root.children.find(component.name);
if (found != contents.root.children.end() && found->second != component.node) {
contents.error = Error{component.name, "Two different components were packed under the same name."};
return;
}
contents.root.children.emplace(std::move(component.name), std::move(component.node));
}
Result<void> write(const std::filesystem::path& path, const Pack& contents, const WriteOptions& options) {
if (contents.error) {
return std::unexpected(*contents.error);
}
std::vector<Payload> payloads;
const Node metadata = detail::provenance(options);
auto saved = collect(metadata, "provenance", options.format, payloads);
if (!saved) {
return saved;
}
for (const auto& [name, node] : contents.root.children) {
if (name == "provenance") {
return std::unexpected(Error{name, "Provenance is written by the archive, not supplied as a model component."});
}
saved = collect(node, name, options.format, payloads);
if (!saved) {
return saved;
}
}
std::ranges::sort(payloads, {}, [](const Payload& payload) { return payload.entry.path; });
Header header{.format = options.format, .version = SERIF_VERSION,
.timestamp = std::get<std::string>(metadata.children.at("written_utc").value),
.index_bytes = print_ledger(payloads).size(), .entries = payloads.size()};
header.data_offset = header.index_offset + header.index_bytes;
std::uint64_t position = header.data_offset;
for (Payload& payload : payloads) {
if (payload.entry.kind != 0) {
position += section_label(payload.entry, options.format).size();
payload.entry.offset = position;
position += payload.bytes.size();
}
}
header.file_bytes = position;
const std::string ledger = print_ledger(payloads);
header.checksum = fingerprint(ledger);
static std::atomic_uint64_t sequence{0};
TemporaryFile temporary;
for (int attempt = 0; attempt < 10 && temporary.stream == nullptr; ++attempt) {
temporary.path = path;
temporary.path += ".tmp-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "-" + std::to_string(sequence++);
temporary.stream = std::fopen(temporary.path.string().c_str(), "wbx");
}
if (temporary.stream == nullptr) {
return std::unexpected(Error{path.string(), "Cannot create an exclusive temporary file beside the destination."});
}
temporary.owned = true;
bool written = temporary.append(print_header(header)) && temporary.append(ledger);
for (const Payload& payload : payloads) {
if (written && payload.entry.kind != 0) {
written = temporary.append(section_label(payload.entry, options.format)) && temporary.append(payload.bytes);
}
}
const int close_status = std::fclose(temporary.stream);
temporary.stream = nullptr;
if (!written || close_status != 0) {
return std::unexpected(Error{path.string(), "The archive could not be written completely; the previous destination was retained."});
}
std::error_code error;
std::filesystem::rename(temporary.path, path, error);
if (error) {
return std::unexpected(Error{path.string(), "Cannot publish the completed archive: " + error.message()});
}
temporary.owned = false;
return {};
}
Result<Reader> Reader::open(const std::filesystem::path& path, ReadOptions options) {
Reader reader;
reader.m_options = options;
reader.m_file.open(path, std::ios::binary | std::ios::ate);
if (!reader.m_file) {
return std::unexpected(Error{path.string(), "Cannot open archive for reading."});
}
const std::streamoff file_size = reader.m_file.tellg();
if (file_size < static_cast<std::streamoff>(detail::header_bytes)) {
return std::unexpected(Error{"header", "Archive is truncated before the complete header."});
}
auto bytes = read_bytes(reader.m_file, 0, detail::header_bytes, "header");
if (!bytes) {
return std::unexpected(bytes.error());
}
auto header = parse_header(*bytes);
if (!header) {
return std::unexpected(header.error());
}
const bool same_major = major_version(header->version) && major_version(header->version) == major_version(SERIF_VERSION);
if ((options.version_tolerance == VersionTolerance::exact && header->version != SERIF_VERSION)
|| (options.version_tolerance == VersionTolerance::same_major && !same_major)) {
return std::unexpected(Error{"header/serif_version", "SERiF " + header->version + " is incompatible with the selected reader tolerance (reader " SERIF_VERSION ")."});
}
if (header->index_offset != detail::header_bytes || header->file_bytes != static_cast<std::uint64_t>(file_size)
|| header->index_bytes > options.maximum_index_bytes || header->index_bytes > header->file_bytes - detail::header_bytes
|| header->data_offset != header->index_offset + header->index_bytes || header->entries > options.maximum_entries) {
std::string cmp_string = std::format("header: [index_offset={}, index_bytes={}, data_offset={}, file_bytes={}, entries={}]", header->index_offset, header->index_bytes, header->data_offset, header->file_bytes, header->entries);
cmp_string = std::format("{}, file: [size={}]", cmp_string, file_size);
cmp_string = std::format("{}, limits: [max_index_bytes={}, max_entries={}]", cmp_string, options.maximum_index_bytes, options.maximum_entries);
cmp_string = std::format("{}, version_tolerance={}", cmp_string, static_cast<int>(options.version_tolerance));
cmp_string = std::format("{}, detail::header_bytes={}", cmp_string, detail::header_bytes);
return std::unexpected(Error{"header", std::format("Invalid file size, section offsets, or configured allocation limits. {}", cmp_string)});
}
auto ledger = read_bytes(reader.m_file, header->index_offset, header->index_bytes, "ledger");
if (!ledger) {
return std::unexpected(ledger.error());
}
if (fingerprint(*ledger) != header->checksum) {
return std::unexpected(Error{"ledger", "Ledger checksum does not match the header."});
}
auto entries = parse_ledger(*ledger, *header, options);
if (!entries) {
return std::unexpected(entries.error());
}
reader.m_entries = std::move(*entries);
reader.m_format = header->format;
reader.m_version = std::move(header->version);
return reader;
}
bool Reader::contains(std::string_view path) const {
const auto found = std::ranges::lower_bound(m_entries, path, {}, &Entry::path);
return found != m_entries.end() && found->path == path;
}
Result<Node> Reader::node(std::string_view path) const {
const auto first = std::ranges::lower_bound(m_entries, path, {}, &Entry::path);
if (first == m_entries.end() || first->path != path) {
return std::unexpected(Error{std::string{path}, "Requested component is not present in the archive ledger."});
}
Node root;
std::map<std::string, Node*> nodes;
const std::string prefix = std::string{path} + "/";
std::uint64_t total_bytes = 0;
for (auto entry = first; entry != m_entries.end(); ++entry) {
if (entry != first && !entry->path.starts_with(prefix)) {
// Sibling names can sort before a parent's descendants (a.b precedes a/b).
if (entry->path < prefix) {
continue;
}
break;
}
if (entry->bytes > m_options.maximum_payload_bytes - total_bytes) {
return std::unexpected(Error{entry->path, "Selected subtree exceeds the configured payload budget."});
}
total_bytes += entry->bytes;
auto bytes = read_bytes(m_file, entry->offset, entry->bytes, entry->path);
if (!bytes) {
return std::unexpected(bytes.error());
}
if (fingerprint(*bytes) != entry->checksum) {
return std::unexpected(Error{entry->path, "Payload checksum mismatch; the component is damaged."});
}
auto value = detail::decode_value(*bytes, *entry, m_format);
if (!value) {
return std::unexpected(value.error());
}
Node* destination = &root;
if (entry != first) {
const std::size_t separator = entry->path.rfind('/');
const auto parent = nodes.find(entry->path.substr(0, separator));
if (parent == nodes.end()) {
return std::unexpected(Error{entry->path, "Requested subtree has a missing parent."});
}
destination = &parent->second->children[entry->path.substr(separator + 1)];
}
*destination = Node{.type = entry->type, .version = entry->version, .value = std::move(*value)};
nodes.emplace(entry->path, destination);
}
return root;
}
Result<void> Reader::validate() const {
for (const Entry& entry : m_entries) {
if (entry.kind != 0) {
auto payload = node(entry.path);
if (!payload) {
return std::unexpected(payload.error());
}
}
}
return {};
}
}