From 8cb28179766edcb4fe34fccee900ced043971293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pa=CC=84vels=20Nadtoc=CC=8Cajevs?= <7645683+bruvzg@users.noreply.github.com> Date: Fri, 25 Apr 2025 19:11:06 +0300 Subject: [PATCH] [PCK] Move directory to the end of file, write exported/saved PCK in place. --- core/io/file_access_pack.cpp | 36 +++--- core/io/file_access_pack.h | 6 +- core/io/pck_packer.cpp | 144 ++++++++++------------ core/io/pck_packer.h | 6 +- doc/classes/PCKPacker.xml | 5 +- editor/export/editor_export_platform.cpp | 146 +++++++++-------------- 6 files changed, 153 insertions(+), 190 deletions(-) diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index 58535ed3eaf..97c563d3a8d 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -256,31 +256,37 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, int64_t pck_start_pos = f->get_position() - 4; + // Read header. uint32_t version = f->get_32(); uint32_t ver_major = f->get_32(); uint32_t ver_minor = f->get_32(); - f->get_32(); // patch number, not used for validation. + uint32_t ver_patch = f->get_32(); // Not used for validation. - ERR_FAIL_COND_V_MSG(version != PACK_FORMAT_VERSION, false, vformat("Pack version unsupported: %d.", version)); - ERR_FAIL_COND_V_MSG(ver_major > GODOT_VERSION_MAJOR || (ver_major == GODOT_VERSION_MAJOR && ver_minor > GODOT_VERSION_MINOR), false, vformat("Pack created with a newer version of the engine: %d.%d.", ver_major, ver_minor)); + ERR_FAIL_COND_V_MSG(version != PACK_FORMAT_VERSION_V3 && version != PACK_FORMAT_VERSION_V2, false, vformat("Pack version unsupported: %d.", version)); + ERR_FAIL_COND_V_MSG(ver_major > GODOT_VERSION_MAJOR || (ver_major == GODOT_VERSION_MAJOR && ver_minor > GODOT_VERSION_MINOR), false, vformat("Pack created with a newer version of the engine: %d.%d.%d.", ver_major, ver_minor, ver_patch)); uint32_t pack_flags = f->get_32(); - uint64_t file_base = f->get_64(); - bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED); - bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); + bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3. - for (int i = 0; i < 16; i++) { - //reserved - f->get_32(); - } - - int file_count = f->get_32(); - - if (rel_filebase) { + uint64_t file_base = f->get_64(); + if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) { file_base += pck_start_pos; } + if (version == PACK_FORMAT_VERSION_V3) { + // V3: Read directory offset and skip reserved part of the header. + uint64_t dir_offset = f->get_64() + pck_start_pos; + f->seek(dir_offset); + } else if (version == PACK_FORMAT_VERSION_V2) { + // V2: Directory directly after the header. + for (int i = 0; i < 16; i++) { + f->get_32(); // Reserved. + } + } + + // Read directory. + int file_count = f->get_32(); if (enc_directory) { Ref fae; fae.instantiate(); @@ -314,7 +320,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, if (flags & PACK_FILE_REMOVAL) { // The file was removed. PackedData::get_singleton()->remove_path(path); } else { - PackedData::get_singleton()->add_path(p_path, path, file_base + ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED)); + PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED)); } } diff --git a/core/io/file_access_pack.h b/core/io/file_access_pack.h index 1d7b59006ba..4c7b78ca1f9 100644 --- a/core/io/file_access_pack.h +++ b/core/io/file_access_pack.h @@ -38,8 +38,12 @@ // Godot's packed file magic header ("GDPC" in ASCII). #define PACK_HEADER_MAGIC 0x43504447 + +#define PACK_FORMAT_VERSION_V2 2 +#define PACK_FORMAT_VERSION_V3 3 + // The current packed file format version number. -#define PACK_FORMAT_VERSION 2 +#define PACK_FORMAT_VERSION PACK_FORMAT_VERSION_V3 enum PackFlags { PACK_DIR_ENCRYPTED = 1 << 0, diff --git a/core/io/pck_packer.cpp b/core/io/pck_packer.cpp index 4c725fa3872..62fb70a3dd1 100644 --- a/core/io/pck_packer.cpp +++ b/core/io/pck_packer.cpp @@ -95,14 +95,28 @@ Error PCKPacker::pck_start(const String &p_pck_path, int p_alignment, const Stri file->store_32(GODOT_VERSION_MINOR); file->store_32(GODOT_VERSION_PATCH); - uint32_t pack_flags = 0; + uint32_t pack_flags = PACK_REL_FILEBASE; if (enc_dir) { pack_flags |= PACK_DIR_ENCRYPTED; } file->store_32(pack_flags); // flags + file_base_ofs = file->get_position(); + file->store_64(0); // Files base. + + dir_base_ofs = file->get_position(); + file->store_64(0); // Directory offset. + + for (int i = 0; i < 16; i++) { + file->store_32(0); // Reserved. + } + + file_base = file->get_position(); + file->seek(file_base_ofs); + file->store_64(file_base); // Update files base. + file->seek(file_base); + files.clear(); - ofs = 0; return OK; } @@ -114,7 +128,7 @@ Error PCKPacker::add_file_removal(const String &p_target_path) { // Simplify path here and on every 'files' access so that paths that have extra '/' // symbols or 'res://' in them still match the MD5 hash for the saved path. pf.path = p_target_path.simplify_path().trim_prefix("res://"); - pf.ofs = ofs; + pf.ofs = file->get_position(); pf.size = 0; pf.removal = true; @@ -138,7 +152,7 @@ Error PCKPacker::add_file(const String &p_target_path, const String &p_source_pa // symbols or 'res://' in them still match the MD5 hash for the saved path. pf.path = p_target_path.simplify_path().trim_prefix("res://"); pf.src_path = p_source_path; - pf.ofs = ofs; + pf.ofs = file->get_position(); pf.size = f->get_length(); Vector data = FileAccess::get_file_as_bytes(p_source_path); @@ -152,18 +166,29 @@ Error PCKPacker::add_file(const String &p_target_path, const String &p_source_pa } pf.encrypted = p_encrypt; - uint64_t _size = pf.size; - if (p_encrypt) { // Add encryption overhead. - if (_size % 16) { // Pad to encryption block size. - _size += 16 - (_size % 16); - } - _size += 16; // hash - _size += 8; // data size - _size += 16; // iv + Ref ftmp = file; + + Ref fae; + if (p_encrypt) { + fae.instantiate(); + ERR_FAIL_COND_V(fae.is_null(), ERR_CANT_CREATE); + + Error err = fae->open_and_parse(file, key, FileAccessEncrypted::MODE_WRITE_AES256, false); + ERR_FAIL_COND_V(err != OK, ERR_CANT_CREATE); + ftmp = fae; } - int pad = _get_pad(alignment, ofs + _size); - ofs = ofs + _size + pad; + ftmp->store_buffer(data); + + if (fae.is_valid()) { + ftmp.unref(); + fae.unref(); + } + + int pad = _get_pad(alignment, file->get_position()); + for (int j = 0; j < pad; j++) { + file->store_8(0); + } files.push_back(pf); @@ -173,14 +198,17 @@ Error PCKPacker::add_file(const String &p_target_path, const String &p_source_pa Error PCKPacker::flush(bool p_verbose) { ERR_FAIL_COND_V_MSG(file.is_null(), ERR_INVALID_PARAMETER, "File must be opened before use."); - int64_t file_base_ofs = file->get_position(); - file->store_64(0); // files base - - for (int i = 0; i < 16; i++) { - file->store_32(0); // reserved + int dir_padding = _get_pad(alignment, file->get_position()); + for (int i = 0; i < dir_padding; i++) { + file->store_8(0); } - // write the index + // Write directory. + uint64_t dir_offset = file->get_position(); + file->seek(dir_base_ofs); + file->store_64(dir_offset); + file->seek(dir_offset); + file->store_32(uint32_t(files.size())); Ref fae; @@ -196,7 +224,8 @@ Error PCKPacker::flush(bool p_verbose) { fhead = fae; } - for (int i = 0; i < files.size(); i++) { + const int file_num = files.size(); + for (int i = 0; i < file_num; i++) { CharString utf8_string = files[i].path.utf8(); int string_len = utf8_string.length(); int pad = _get_pad(4, string_len); @@ -207,9 +236,9 @@ Error PCKPacker::flush(bool p_verbose) { fhead->store_8(0); } - fhead->store_64(files[i].ofs); - fhead->store_64(files[i].size); // pay attention here, this is where file is - fhead->store_buffer(files[i].md5.ptr(), 16); //also save md5 for file + fhead->store_64(files[i].ofs - file_base); + fhead->store_64(files[i].size); + fhead->store_buffer(files[i].md5.ptr(), 16); uint32_t flags = 0; if (files[i].encrypted) { @@ -219,6 +248,10 @@ Error PCKPacker::flush(bool p_verbose) { flags |= PACK_FILE_REMOVAL; } fhead->store_32(flags); + + if (p_verbose) { + print_line(vformat("[%d/%d - %d%%] PCKPacker flush: %s -> %s", i, file_num, float(i) / file_num * 100, files[i].src_path, files[i].path)); + } } if (fae.is_valid()) { @@ -226,63 +259,12 @@ Error PCKPacker::flush(bool p_verbose) { fae.unref(); } - int header_padding = _get_pad(alignment, file->get_position()); - for (int i = 0; i < header_padding; i++) { - file->store_8(0); - } - - uint64_t file_base = file->get_position(); - file->seek(file_base_ofs); - file->store_64(file_base); // update files base - file->seek(file_base); - - const uint32_t buf_max = 65536; - uint8_t *buf = memnew_arr(uint8_t, buf_max); - - int count = 0; - for (int i = 0; i < files.size(); i++) { - if (files[i].removal) { - continue; - } - - Ref src = FileAccess::open(files[i].src_path, FileAccess::READ); - uint64_t to_write = files[i].size; - - Ref ftmp = file; - if (files[i].encrypted) { - fae.instantiate(); - ERR_FAIL_COND_V(fae.is_null(), ERR_CANT_CREATE); - - Error err = fae->open_and_parse(file, key, FileAccessEncrypted::MODE_WRITE_AES256, false); - ERR_FAIL_COND_V(err != OK, ERR_CANT_CREATE); - ftmp = fae; - } - - while (to_write > 0) { - uint64_t read = src->get_buffer(buf, MIN(to_write, buf_max)); - ftmp->store_buffer(buf, read); - to_write -= read; - } - - if (fae.is_valid()) { - ftmp.unref(); - fae.unref(); - } - - int pad = _get_pad(alignment, file->get_position()); - for (int j = 0; j < pad; j++) { - file->store_8(0); - } - - count += 1; - const int file_num = files.size(); - if (p_verbose && (file_num > 0)) { - print_line(vformat("[%d/%d - %d%%] PCKPacker flush: %s -> %s", count, file_num, float(count) / file_num * 100, files[i].src_path, files[i].path)); - } - } - file.unref(); - memdelete_arr(buf); - return OK; } + +PCKPacker::~PCKPacker() { + if (file.is_valid()) { + flush(); + } +} diff --git a/core/io/pck_packer.h b/core/io/pck_packer.h index 4c386b36a28..c7a2b75995c 100644 --- a/core/io/pck_packer.h +++ b/core/io/pck_packer.h @@ -39,11 +39,14 @@ class PCKPacker : public RefCounted { Ref file; int alignment = 0; - uint64_t ofs = 0; Vector key; bool enc_dir = false; + uint64_t file_base = 0; + uint64_t file_base_ofs = 0; + uint64_t dir_base_ofs = 0; + static void _bind_methods(); struct File { @@ -64,4 +67,5 @@ public: Error flush(bool p_verbose = false); PCKPacker() {} + ~PCKPacker(); }; diff --git a/doc/classes/PCKPacker.xml b/doc/classes/PCKPacker.xml index dbb978865c6..90b9be6cd10 100644 --- a/doc/classes/PCKPacker.xml +++ b/doc/classes/PCKPacker.xml @@ -31,7 +31,7 @@ - Adds the [param source_path] file to the current PCK package at the [param target_path] internal path. The [code]res://[/code] prefix for [param target_path] is optional and stripped internally. + Adds the [param source_path] file to the current PCK package at the [param target_path] internal path. The [code]res://[/code] prefix for [param target_path] is optional and stripped internally. File content is immediately written to the PCK. @@ -45,7 +45,8 @@ - Writes the files specified using all [method add_file] calls since the last flush. If [param verbose] is [code]true[/code], a list of files added will be printed to the console for easier debugging. + Writes the file directory and closes the PCK. If [param verbose] is [code]true[/code], a list of files added will be printed to the console for easier debugging. + [b]Note:[/b] [PCKPacker] will automatically flush when it's freed, which happens when it goes out of scope or when it gets assigned with [code]null[/code]. In C# the reference must be disposed after use, either with the [code]using[/code] statement or by calling the [code]Dispose[/code] method directly. diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index 8f877056b68..040c777c1ee 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -305,10 +305,10 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa } fae.instantiate(); - ERR_FAIL_COND_V(fae.is_null(), ERR_SKIP); + ERR_FAIL_COND_V(fae.is_null(), ERR_FILE_CANT_OPEN); Error err = fae->open_and_parse(ftmp, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); - ERR_FAIL_COND_V(err != OK, ERR_SKIP); + ERR_FAIL_COND_V(err != OK, ERR_FILE_CANT_OPEN); ftmp = fae; } @@ -320,6 +320,8 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa fae.unref(); } + ERR_FAIL_COND_V(pd->f->get_position() - sd.ofs < (uint64_t)p_data.size(), ERR_FILE_CANT_WRITE); + int pad = _get_pad(PCK_PADDING, pd->f->get_position()); for (int i = 0; i < pad; i++) { pd->f->store_8(0); @@ -1906,53 +1908,19 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b Ref da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); da->make_dir_recursive(EditorPaths::get_singleton()->get_temp_dir()); - String tmppath = EditorPaths::get_singleton()->get_temp_dir().path_join("packtmp"); - Ref ftmp = FileAccess::open(tmppath, FileAccess::WRITE); - if (ftmp.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Cannot create file \"%s\"."), tmppath)); - return ERR_CANT_CREATE; - } - - PackData pd; - pd.ep = &ep; - pd.f = ftmp; - pd.so_files = p_so_files; - - Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object); - - // Close temp file. - pd.f.unref(); - ftmp.unref(); - - if (err != OK) { - DirAccess::remove_file_or_error(tmppath); - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Failed to export project files.")); - return err; - } - - if (pd.file_ofs.is_empty()) { - DirAccess::remove_file_or_error(tmppath); - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export.")); - return FAILED; - } - - pd.file_ofs.sort(); //do sort, so we can do binary search later - Ref f; int64_t embed_pos = 0; if (!p_embed) { - // Regular output to separate PCK file + // Regular output to separate PCK file. f = FileAccess::open(p_path, FileAccess::WRITE); if (f.is_null()) { - DirAccess::remove_file_or_error(tmppath); add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for writing at path \"%s\"."), p_path)); return ERR_CANT_CREATE; } } else { - // Append to executable + // Append to executable. f = FileAccess::open(p_path, FileAccess::READ_WRITE); if (f.is_null()) { - DirAccess::remove_file_or_error(tmppath); add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for reading-writing at path \"%s\"."), p_path)); return ERR_FILE_CANT_OPEN; } @@ -1973,32 +1941,68 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b int64_t pck_start_pos = f->get_position(); + // Write header. f->store_32(PACK_HEADER_MAGIC); f->store_32(PACK_FORMAT_VERSION); f->store_32(GODOT_VERSION_MAJOR); f->store_32(GODOT_VERSION_MINOR); f->store_32(GODOT_VERSION_PATCH); - uint32_t pack_flags = 0; + uint32_t pack_flags = PACK_REL_FILEBASE; bool enc_pck = p_preset->get_enc_pck(); bool enc_directory = p_preset->get_enc_directory(); if (enc_pck && enc_directory) { pack_flags |= PACK_DIR_ENCRYPTED; } - if (p_embed) { - pack_flags |= PACK_REL_FILEBASE; - } - f->store_32(pack_flags); // flags + f->store_32(pack_flags); // Flags. uint64_t file_base_ofs = f->get_position(); - f->store_64(0); // files base + f->store_64(0); // Files base. + + uint64_t dir_base_ofs = f->get_position(); + f->store_64(0); // Directory offset. for (int i = 0; i < 16; i++) { - //reserved - f->store_32(0); + f->store_32(0); // Reserved. } - f->store_32(pd.file_ofs.size()); //amount of files + uint64_t file_base = f->get_position(); + f->seek(file_base_ofs); + f->store_64(file_base - pck_start_pos); // Update files base. + f->seek(file_base); + + // Write files. + PackData pd; + pd.ep = &ep; + pd.f = f; + pd.so_files = p_so_files; + + Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object); + + if (err != OK) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Failed to export project files.")); + return err; + } + + if (pd.file_ofs.is_empty()) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export.")); + return FAILED; + } + + pd.file_ofs.sort(); // Do sort, so we can do binary search later (where ?). + + int dir_padding = _get_pad(PCK_PADDING, f->get_position()); + for (int i = 0; i < dir_padding; i++) { + f->store_8(0); + } + + // Write directory. + uint64_t dir_offset = f->get_position(); + f->seek(dir_base_ofs); + f->store_64(dir_offset - pck_start_pos); + f->seek(dir_offset); + + f->store_32(pd.file_ofs.size()); Ref fae; Ref fhead = f; @@ -2048,7 +2052,7 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b for (int64_t j = 0; j < pd.file_ofs[i].md5.size(); j++) { seed = ((seed << 5) + seed) ^ pd.file_ofs[i].md5[j]; } - seed = ((seed << 5) + seed) ^ pd.file_ofs[i].ofs; + seed = ((seed << 5) + seed) ^ (pd.file_ofs[i].ofs - file_base); seed = ((seed << 5) + seed) ^ pd.file_ofs[i].size; } @@ -2078,9 +2082,9 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b fhead->store_8(0); } - fhead->store_64(pd.file_ofs[i].ofs); - fhead->store_64(pd.file_ofs[i].size); // pay attention here, this is where file is - fhead->store_buffer(pd.file_ofs[i].md5.ptr(), 16); //also save md5 for file + fhead->store_64(pd.file_ofs[i].ofs - file_base); + fhead->store_64(pd.file_ofs[i].size); + fhead->store_buffer(pd.file_ofs[i].md5.ptr(), 16); uint32_t flags = 0; if (pd.file_ofs[i].encrypted) { flags |= PACK_FILE_ENCRYPTED; @@ -2096,44 +2100,8 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b fae.unref(); } - int header_padding = _get_pad(PCK_PADDING, f->get_position()); - for (int i = 0; i < header_padding; i++) { - f->store_8(0); - } - - uint64_t file_base = f->get_position(); - uint64_t file_base_store = file_base; - if (pack_flags & PACK_REL_FILEBASE) { - file_base_store -= pck_start_pos; - } - f->seek(file_base_ofs); - f->store_64(file_base_store); // update files base - f->seek(file_base); - - // Save the rest of the data. - - ftmp = FileAccess::open(tmppath, FileAccess::READ); - if (ftmp.is_null()) { - DirAccess::remove_file_or_error(tmppath); - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file to read from path \"%s\"."), tmppath)); - return ERR_CANT_CREATE; - } - - const int bufsize = 16384; - uint8_t buf[bufsize]; - - while (true) { - uint64_t got = ftmp->get_buffer(buf, bufsize); - if (got == 0) { - break; - } - f->store_buffer(buf, got); - } - - ftmp.unref(); // Close temp file. - if (p_embed) { - // Ensure embedded data ends at a 64-bit multiple + // Ensure embedded data ends at a 64-bit multiple. uint64_t embed_end = f->get_position() - embed_pos + 12; uint64_t pad = embed_end % 8; for (uint64_t i = 0; i < pad; i++) { @@ -2150,8 +2118,6 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b } f->close(); - DirAccess::remove_file_or_error(tmppath); - return OK; }