From 42733a2a5cda8ac41567008a2f2f11ced83181b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pa=CC=84vels=20Nadtoc=CC=8Cajevs?= <7645683+bruvzg@users.noreply.github.com> Date: Thu, 20 Feb 2025 11:08:16 +0200 Subject: [PATCH] Implement sparse bundle PCK support. --- core/config/project_settings.cpp | 7 + core/io/file_access.cpp | 4 +- core/io/file_access.h | 1 + core/io/file_access_pack.cpp | 26 +- core/io/file_access_pack.h | 4 +- editor/export/editor_export_platform.cpp | 271 ++++++++++-------- editor/export/editor_export_platform.h | 10 +- platform/android/export/export_plugin.cpp | 112 +++++++- platform/android/export/export_plugin.h | 3 + .../android/export/gradle_export_util.cpp | 54 +++- platform/android/export/gradle_export_util.h | 5 + 11 files changed, 351 insertions(+), 146 deletions(-) diff --git a/core/config/project_settings.cpp b/core/config/project_settings.cpp index b646945eae9..a9673cf79b3 100644 --- a/core/config/project_settings.cpp +++ b/core/config/project_settings.cpp @@ -614,6 +614,8 @@ void ProjectSettings::_convert_to_last_version(int p_from_version) { * appending '.pck' to the binary name (e.g. 'linux_game' -> 'linux_game.pck'). * o PCK with the same basename as the binary in the current working directory. * Same as above for the two possible PCK file names. + * - On Android, look for 'assets.sparsepck' and try loading it, if it doesn't work, + * proceed to the next step. * - On relevant platforms (Android/iOS), lookup project file in OS resource path. * If found, load it or fail. * - Lookup project file in passed `p_path` (--path passed by the user), i.e. we @@ -701,6 +703,11 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b } } +#ifdef ANDROID_ENABLED + // Attempt to load sparse PCK assets. + _load_resource_pack("res://assets.sparsepck", false, 0, true); +#endif + // Try to use the filesystem for files, according to OS. // (Only Android -when reading from pck- and iOS use this.) diff --git a/core/io/file_access.cpp b/core/io/file_access.cpp index 5628d86ec81..47cfb2d57f7 100644 --- a/core/io/file_access.cpp +++ b/core/io/file_access.cpp @@ -159,7 +159,7 @@ Ref FileAccess::open(const String &p_path, int p_mode_flags, Error * //try packed data first Ref ret; - if (!(p_mode_flags & WRITE) && PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled()) { + if (!(p_mode_flags & WRITE) && !(p_mode_flags & SKIP_PACK) && PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled()) { ret = PackedData::get_singleton()->try_open_path(p_path); if (ret.is_valid()) { if (r_error) { @@ -170,7 +170,7 @@ Ref FileAccess::open(const String &p_path, int p_mode_flags, Error * } ret = create_for_path(p_path); - Error err = ret->open_internal(p_path, p_mode_flags); + Error err = ret->open_internal(p_path, p_mode_flags & ~SKIP_PACK); if (r_error) { *r_error = err; diff --git a/core/io/file_access.h b/core/io/file_access.h index 30f27218957..ed306d6cc81 100644 --- a/core/io/file_access.h +++ b/core/io/file_access.h @@ -58,6 +58,7 @@ public: WRITE = 2, READ_WRITE = 3, WRITE_READ = 7, + SKIP_PACK = 16, }; enum UnixPermissionFlags : int32_t { diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index 24e63489425..244469e9d90 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -45,7 +45,7 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t return ERR_FILE_UNRECOGNIZED; } -void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted) { +void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle) { String simplified_path = p_path.simplify_path().trim_prefix("res://"); PathMD5 pmd5(simplified_path.md5_buffer()); @@ -53,6 +53,7 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64 PackedFile pf; pf.encrypted = p_encrypted; + pf.bundle = p_bundle; pf.pack = p_pkg_path; pf.offset = p_ofs; pf.size = p_size; @@ -268,6 +269,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, uint32_t pack_flags = f->get_32(); bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED); bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3. + bool sparse_bundle = (pack_flags & PACK_SPARSE_BUNDLE); uint64_t file_base = f->get_64(); if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) { @@ -320,7 +322,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, if (flags & PACK_FILE_REMOVAL) { // The file was removed. PackedData::get_singleton()->remove_path(path); } else { - PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED)); + PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle); } } @@ -360,7 +362,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f for (const String &file_name : da->get_files()) { String file_path = p_path.path_join(file_name); uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false); + PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false); } for (const String &sub_dir_name : da->get_directories()) { @@ -467,13 +469,19 @@ void FileAccessPack::close() { f = Ref(); } -FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) : - pf(p_file), - f(FileAccess::open(pf.pack, FileAccess::READ)) { - ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack))); +FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) { + pf = p_file; + if (pf.bundle) { + String simplified_path = p_path.simplify_path(); + f = FileAccess::open(simplified_path, FileAccess::READ | FileAccess::SKIP_PACK); + off = 0; // For the sparse pack offset is always zero. + } else { + f = FileAccess::open(pf.pack, FileAccess::READ); + f->seek(pf.offset); + off = pf.offset; + } - f->seek(pf.offset); - off = pf.offset; + ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack))); if (pf.encrypted) { Ref fae; diff --git a/core/io/file_access_pack.h b/core/io/file_access_pack.h index 4c7b78ca1f9..c2c7575542c 100644 --- a/core/io/file_access_pack.h +++ b/core/io/file_access_pack.h @@ -48,6 +48,7 @@ enum PackFlags { PACK_DIR_ENCRYPTED = 1 << 0, PACK_REL_FILEBASE = 1 << 1, + PACK_SPARSE_BUNDLE = 1 << 2, }; enum PackFileFlags { @@ -70,6 +71,7 @@ public: uint8_t md5[16]; PackSource *src = nullptr; bool encrypted; + bool bundle; }; private: @@ -114,7 +116,7 @@ private: public: void add_pack_source(PackSource *p_source); - void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false); // for PackSource + void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false); // for PackSource void remove_path(const String &p_path); uint8_t *get_file_hash(const String &p_path); HashSet get_file_paths() const; diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index ca779a9d935..4fce70877bb 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -252,41 +252,25 @@ void EditorExportPlatform::_unload_patches() { PackedData::get_singleton()->clear(); } -Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed) { - ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export."); - - PackData *pd = (PackData *)p_userdata; - - String simplified_path = p_path.simplify_path(); - if (simplified_path.begins_with("uid://")) { - simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path(); - print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path)); - } - - SavedData sd; - sd.path_utf8 = simplified_path.trim_prefix("res://").utf8(); - sd.ofs = pd->f->get_position(); - sd.size = p_data.size(); - sd.encrypted = false; - +Error EditorExportPlatform::_encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt) { + r_encrypt = false; for (int i = 0; i < p_enc_in_filters.size(); ++i) { - if (simplified_path.matchn(p_enc_in_filters[i]) || simplified_path.trim_prefix("res://").matchn(p_enc_in_filters[i])) { - sd.encrypted = true; + if (p_path.matchn(p_enc_in_filters[i]) || p_path.trim_prefix("res://").matchn(p_enc_in_filters[i])) { + r_encrypt = true; break; } } for (int i = 0; i < p_enc_ex_filters.size(); ++i) { - if (simplified_path.matchn(p_enc_ex_filters[i]) || simplified_path.trim_prefix("res://").matchn(p_enc_ex_filters[i])) { - sd.encrypted = false; + if (p_path.matchn(p_enc_ex_filters[i]) || p_path.trim_prefix("res://").matchn(p_enc_ex_filters[i])) { + r_encrypt = false; break; } } Ref fae; - Ref ftmp = pd->f; - - if (sd.encrypted) { + Ref ftmp = p_fd; + if (r_encrypt) { Vector iv; if (p_seed != 0) { uint64_t seed = p_seed; @@ -319,12 +303,44 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa ftmp.unref(); fae.unref(); } + return OK; +} - ERR_FAIL_COND_V(pd->f->get_position() - sd.ofs < (uint64_t)p_data.size(), ERR_FILE_CANT_WRITE); +Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed) { + ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export."); - int pad = _get_pad(PCK_PADDING, pd->f->get_position()); - for (int i = 0; i < pad; i++) { - pd->f->store_8(0); + PackData *pd = (PackData *)p_userdata; + + String simplified_path = p_path.simplify_path(); + if (simplified_path.begins_with("uid://")) { + simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path(); + print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path)); + } + + Ref ftmp; + if (pd->use_sparse_pck) { + ftmp = FileAccess::open(pd->path.get_base_dir().path_join(simplified_path.trim_prefix("res://")), FileAccess::WRITE); + } else { + ftmp = pd->f; + } + + SavedData sd; + sd.path_utf8 = simplified_path.trim_prefix("res://").utf8(); + sd.ofs = (pd->use_sparse_pck) ? 0 : pd->f->get_position(); + sd.size = p_data.size(); + Error err = _encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted); + if (err != OK) { + return err; + } + if (!pd->use_sparse_pck) { + ERR_FAIL_COND_V(pd->f->get_position() - sd.ofs < (uint64_t)p_data.size(), ERR_FILE_CANT_WRITE); + } + + if (!pd->use_sparse_pck) { + int pad = _get_pad(PCK_PADDING, pd->f->get_position()); + for (int i = 0; i < pad; i++) { + pd->f->store_8(0); + } } // Store MD5 of original file. @@ -1897,6 +1913,105 @@ Dictionary EditorExportPlatform::_save_zip_patch(const Ref & return ret; } +bool EditorExportPlatform::_store_header(Ref p_fd, bool p_enc, bool p_sparse, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs) { + p_fd->store_32(PACK_HEADER_MAGIC); + p_fd->store_32(PACK_FORMAT_VERSION); + p_fd->store_32(GODOT_VERSION_MAJOR); + p_fd->store_32(GODOT_VERSION_MINOR); + p_fd->store_32(GODOT_VERSION_PATCH); + + uint32_t pack_flags = PACK_REL_FILEBASE; + if (p_enc) { + pack_flags |= PACK_DIR_ENCRYPTED; + } + if (p_sparse) { + pack_flags |= PACK_SPARSE_BUNDLE; + } + p_fd->store_32(pack_flags); // Flags. + + r_file_base_ofs = p_fd->get_position(); + p_fd->store_64(0); // Files base offset. + + r_dir_base_ofs = p_fd->get_position(); + p_fd->store_64(0); // Directory offset. + + for (int i = 0; i < 16; i++) { + //reserved + p_fd->store_32(0); + } + return true; +} + +bool EditorExportPlatform::_encrypt_and_store_directory(Ref p_fd, PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base) { + Ref fae; + Ref fhead = p_fd; + + fhead->store_32(p_pack_data.file_ofs.size()); //amount of files + + if (!p_key.is_empty()) { + uint64_t seed = p_seed; + fae.instantiate(); + if (fae.is_null()) { + return false; + } + + Vector iv; + if (seed != 0) { + for (int i = 0; i < p_pack_data.file_ofs.size(); i++) { + for (int64_t j = 0; j < p_pack_data.file_ofs[i].path_utf8.length(); j++) { + seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].path_utf8.get_data()[j]; + } + for (int64_t j = 0; j < p_pack_data.file_ofs[i].md5.size(); j++) { + seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].md5[j]; + } + seed = ((seed << 5) + seed) ^ (p_pack_data.file_ofs[i].ofs - p_file_base); + seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].size; + } + + RandomPCG rng = RandomPCG(seed); + iv.resize(16); + for (int i = 0; i < 16; i++) { + iv.write[i] = rng.rand() % 256; + } + } + + Error err = fae->open_and_parse(fhead, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); + if (err != OK) { + return false; + } + + fhead = fae; + } + for (int i = 0; i < p_pack_data.file_ofs.size(); i++) { + uint32_t string_len = p_pack_data.file_ofs[i].path_utf8.length(); + uint32_t pad = _get_pad(4, string_len); + + fhead->store_32(string_len + pad); + fhead->store_buffer((const uint8_t *)p_pack_data.file_ofs[i].path_utf8.get_data(), string_len); + for (uint32_t j = 0; j < pad; j++) { + fhead->store_8(0); + } + + fhead->store_64(p_pack_data.file_ofs[i].ofs - p_file_base); + fhead->store_64(p_pack_data.file_ofs[i].size); // pay attention here, this is where file is + fhead->store_buffer(p_pack_data.file_ofs[i].md5.ptr(), 16); //also save md5 for file + uint32_t flags = 0; + if (p_pack_data.file_ofs[i].encrypted) { + flags |= PACK_FILE_ENCRYPTED; + } + if (p_pack_data.file_ofs[i].removal) { + flags |= PACK_FILE_REMOVAL; + } + fhead->store_32(flags); + } + + if (fae.is_valid()) { + fhead.unref(); + fae.unref(); + } + return true; +} + Error EditorExportPlatform::save_pack(const Ref &p_preset, bool p_debug, const String &p_path, Vector *p_so_files, EditorExportSaveFunction p_save_func, EditorExportRemoveFunction p_remove_func, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) { EditorProgress ep("savepack", TTR("Packing"), 102, true); @@ -1940,31 +2055,10 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b } int64_t pck_start_pos = f->get_position(); + uint64_t file_base_ofs = 0; + uint64_t dir_base_ofs = 0; - // Write header. - f->store_32(PACK_HEADER_MAGIC); - f->store_32(PACK_FORMAT_VERSION); - f->store_32(GODOT_VERSION_MAJOR); - f->store_32(GODOT_VERSION_MINOR); - f->store_32(GODOT_VERSION_PATCH); - - uint32_t pack_flags = PACK_REL_FILEBASE; - bool enc_pck = p_preset->get_enc_pck(); - bool enc_directory = p_preset->get_enc_directory(); - if (enc_pck && enc_directory) { - pack_flags |= PACK_DIR_ENCRYPTED; - } - f->store_32(pack_flags); // Flags. - - uint64_t file_base_ofs = f->get_position(); - f->store_64(0); // Files base. - - uint64_t dir_base_ofs = f->get_position(); - f->store_64(0); // Directory offset. - - for (int i = 0; i < 16; i++) { - f->store_32(0); // Reserved. - } + _store_header(f, p_preset->get_enc_pck() && p_preset->get_enc_directory(), false, file_base_ofs, dir_base_ofs); // Align for first file. int file_padding = _get_pad(PCK_PADDING, f->get_position()); @@ -1982,6 +2076,7 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b pd.ep = &ep; pd.f = f; pd.so_files = p_so_files; + pd.path = p_path; Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object); @@ -2008,15 +2103,9 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b f->store_64(dir_offset - pck_start_pos); f->seek(dir_offset); - f->store_32(pd.file_ofs.size()); - - Ref fae; - Ref fhead = f; - - if (enc_pck && enc_directory) { - uint64_t seed = p_preset->get_seed(); + Vector key; + if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { String script_key = _get_script_encryption_key(p_preset); - Vector key; key.resize(32); if (script_key.length() == 64) { for (int i = 0; i < 32; i++) { @@ -2043,67 +2132,11 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b key.write[i] = v; } } - fae.instantiate(); - if (fae.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); - return ERR_CANT_CREATE; - } - - Vector iv; - if (seed != 0) { - for (int i = 0; i < pd.file_ofs.size(); i++) { - for (int64_t j = 0; j < pd.file_ofs[i].path_utf8.length(); j++) { - seed = ((seed << 5) + seed) ^ pd.file_ofs[i].path_utf8.get_data()[j]; - } - for (int64_t j = 0; j < pd.file_ofs[i].md5.size(); j++) { - seed = ((seed << 5) + seed) ^ pd.file_ofs[i].md5[j]; - } - seed = ((seed << 5) + seed) ^ (pd.file_ofs[i].ofs - file_base); - seed = ((seed << 5) + seed) ^ pd.file_ofs[i].size; - } - - RandomPCG rng = RandomPCG(seed); - iv.resize(16); - for (int i = 0; i < 16; i++) { - iv.write[i] = rng.rand() % 256; - } - } - - err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); - if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't open encrypted file to write.")); - return ERR_CANT_CREATE; - } - - fhead = fae; } - for (int i = 0; i < pd.file_ofs.size(); i++) { - uint32_t string_len = pd.file_ofs[i].path_utf8.length(); - uint32_t pad = _get_pad(4, string_len); - - fhead->store_32(string_len + pad); - fhead->store_buffer((const uint8_t *)pd.file_ofs[i].path_utf8.get_data(), string_len); - for (uint32_t j = 0; j < pad; j++) { - fhead->store_8(0); - } - - fhead->store_64(pd.file_ofs[i].ofs - file_base); - fhead->store_64(pd.file_ofs[i].size); - fhead->store_buffer(pd.file_ofs[i].md5.ptr(), 16); - uint32_t flags = 0; - if (pd.file_ofs[i].encrypted) { - flags |= PACK_FILE_ENCRYPTED; - } - if (pd.file_ofs[i].removal) { - flags |= PACK_FILE_REMOVAL; - } - fhead->store_32(flags); - } - - if (fae.is_valid()) { - fhead.unref(); - fae.unref(); + if (!_encrypt_and_store_directory(f, pd, key, p_preset->get_seed(), file_base)) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); + return ERR_CANT_CREATE; } if (p_embed) { diff --git a/editor/export/editor_export_platform.h b/editor/export/editor_export_platform.h index 19f6256931e..8c3488ab05a 100644 --- a/editor/export/editor_export_platform.h +++ b/editor/export/editor_export_platform.h @@ -78,7 +78,6 @@ public: String text; }; -private: struct SavedData { uint64_t ofs = 0; uint64_t size = 0; @@ -93,12 +92,20 @@ private: }; struct PackData { + String path; Ref f; Vector file_ofs; EditorProgress *ep = nullptr; Vector *so_files = nullptr; + bool use_sparse_pck = false; }; + static bool _store_header(Ref p_fd, bool p_enc, bool p_sparse, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs); + static bool _encrypt_and_store_directory(Ref p_fd, PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base); + static Error _encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt); + String _get_script_encryption_key(const Ref &p_preset) const; + +private: struct ZipData { void *zip = nullptr; EditorProgress *ep = nullptr; @@ -151,7 +158,6 @@ private: bool _is_editable_ancestor(Node *p_root, Node *p_node); String _export_customize(const String &p_path, LocalVector> &customize_resources_plugins, LocalVector> &customize_scenes_plugins, HashMap &export_cache, const String &export_base_path, bool p_force_save); - String _get_script_encryption_key(const Ref &p_preset) const; protected: struct ExportNotifier { diff --git a/platform/android/export/export_plugin.cpp b/platform/android/export/export_plugin.cpp index a68da744977..b7958842210 100644 --- a/platform/android/export/export_plugin.cpp +++ b/platform/android/export/export_plugin.cpp @@ -625,6 +625,7 @@ bool EditorExportPlatformAndroid::_should_compress_asset(const String &p_path, c ".cfb", // Don't let small config files slow-down startup ".scn", // Binary scenes are usually already compressed ".ctex", // Streamable textures are usually already compressed + ".pck", // Pack. // Trailer for easier processing nullptr }; @@ -800,14 +801,25 @@ Error EditorExportPlatformAndroid::save_apk_so(void *p_userdata, const SharedObj Error EditorExportPlatformAndroid::save_apk_file(void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed) { APKExportData *ed = static_cast(p_userdata); - String path = p_path.simplify_path(); - if (path.begins_with("uid://")) { - path = ResourceUID::uid_to_path(path).simplify_path(); - print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, path)); + String simplified_path = p_path.simplify_path(); + if (simplified_path.begins_with("uid://")) { + simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path(); + print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path)); } - const String dst_path = path.replace_first("res://", "assets/"); - store_in_apk(ed, dst_path, p_data, _should_compress_asset(path, p_data) ? Z_DEFLATED : 0); + Vector enc_data; + EditorExportPlatform::SavedData sd; + Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, enc_data, sd); + if (err != OK) { + return err; + } + + const String dst_path = String("assets/") + simplified_path.trim_prefix("res://"); + print_verbose("Saving project files from " + simplified_path + " into " + dst_path); + store_in_apk(ed, dst_path, enc_data, _should_compress_asset(simplified_path, enc_data) ? Z_DEFLATED : 0); + + ed->pd.file_ofs.push_back(sd); + return OK; } @@ -3409,6 +3421,68 @@ Error EditorExportPlatformAndroid::export_project(const Ref return export_project_helper(p_preset, p_debug, p_path, export_format, should_sign, p_flags); } +Error EditorExportPlatformAndroid::_generate_sparse_pck_metadata(const Ref &p_preset, PackData &p_pack_data, Vector &r_data) { + Error err; + Ref ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export_index", "tmp", false, &err); + if (err != OK) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not create temporary file!")); + return err; + } + int64_t pck_start_pos = ftmp->get_position(); + uint64_t file_base_ofs = 0; + uint64_t dir_base_ofs = 0; + EditorExportPlatform::_store_header(ftmp, p_preset->get_enc_pck() && p_preset->get_enc_directory(), true, file_base_ofs, dir_base_ofs); + + // Write directory. + uint64_t dir_offset = ftmp->get_position(); + ftmp->seek(dir_base_ofs); + ftmp->store_64(dir_offset - pck_start_pos); + ftmp->seek(dir_offset); + + Vector key; + if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { + String script_key = _get_script_encryption_key(p_preset); + key.resize(32); + if (script_key.length() == 64) { + for (int i = 0; i < 32; i++) { + int v = 0; + if (i * 2 < script_key.length()) { + char32_t ct = script_key[i * 2]; + if (is_digit(ct)) { + ct = ct - '0'; + } else if (ct >= 'a' && ct <= 'f') { + ct = 10 + ct - 'a'; + } + v |= ct << 4; + } + + if (i * 2 + 1 < script_key.length()) { + char32_t ct = script_key[i * 2 + 1]; + if (is_digit(ct)) { + ct = ct - '0'; + } else if (ct >= 'a' && ct <= 'f') { + ct = 10 + ct - 'a'; + } + v |= ct; + } + key.write[i] = v; + } + } + } + + if (!EditorExportPlatform::_encrypt_and_store_directory(ftmp, p_pack_data, key, p_preset->get_seed(), 0)) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); + return ERR_CANT_CREATE; + } + + r_data.resize(ftmp->get_length()); + ftmp->seek(0); + ftmp->get_buffer(r_data.ptrw(), r_data.size()); + ftmp.unref(); + + return OK; +} + Error EditorExportPlatformAndroid::export_project_helper(const Ref &p_preset, bool p_debug, const String &p_path, int export_format, bool should_sign, BitField p_flags) { ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags); @@ -3536,7 +3610,22 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref enc_data; + err = _generate_sparse_pck_metadata(p_preset, user_data.pd, enc_data); + if (err != OK) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!")); + return err; + } + + err = store_file_at_path(user_data.assets_directory + "/assets.sparsepck", enc_data); + if (err != OK) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not write PCK directory!")); + return err; + } } if (err != OK) { add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not export project files to gradle project.")); @@ -3988,7 +4077,18 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref enc_data; + err = _generate_sparse_pck_metadata(p_preset, ed.pd, enc_data); + if (err != OK) { + add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!")); + return err; + } + + store_in_apk(&ed, "assets/assets.sparsepck", enc_data, 0); } } diff --git a/platform/android/export/export_plugin.h b/platform/android/export/export_plugin.h index 0620c9f966e..a17c4374141 100644 --- a/platform/android/export/export_plugin.h +++ b/platform/android/export/export_plugin.h @@ -73,6 +73,7 @@ class EditorExportPlatformAndroid : public EditorExportPlatform { }; struct APKExportData { + EditorExportPlatform::PackData pd; zipFile apk; EditorProgress *ep = nullptr; }; @@ -193,6 +194,8 @@ class EditorExportPlatformAndroid : public EditorExportPlatform { bool _uses_vulkan(const Ref &p_preset) const; + Error _generate_sparse_pck_metadata(const Ref &p_preset, PackData &p_pack_data, Vector &r_data); + protected: void _notification(int p_what); diff --git a/platform/android/export/gradle_export_util.cpp b/platform/android/export/gradle_export_util.cpp index 26d12884ba4..5175a69d240 100644 --- a/platform/android/export/gradle_export_util.cpp +++ b/platform/android/export/gradle_export_util.cpp @@ -172,15 +172,24 @@ Error store_string_at_path(const String &p_path, const String &p_data) { Error rename_and_store_file_in_gradle_project(void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed) { CustomExportData *export_data = static_cast(p_userdata); - String path = p_path.simplify_path(); - if (path.begins_with("uid://")) { - path = ResourceUID::uid_to_path(path).simplify_path(); - print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, path)); + String simplified_path = p_path.simplify_path(); + if (simplified_path.begins_with("uid://")) { + simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path(); + print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path)); } - const String dst_path = path.replace_first("res://", export_data->assets_directory + "/"); - print_verbose("Saving project files from " + path + " into " + dst_path); - Error err = store_file_at_path(dst_path, p_data); + Vector enc_data; + EditorExportPlatform::SavedData sd; + Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, enc_data, sd); + if (err != OK) { + return err; + } + + const String dst_path = export_data->assets_directory + String("/") + simplified_path.trim_prefix("res://"); + print_verbose("Saving project files from " + simplified_path + " into " + dst_path); + err = store_file_at_path(dst_path, enc_data); + + export_data->pd.file_ofs.push_back(sd); return err; } @@ -354,3 +363,34 @@ String _get_application_tag(const Ref &p_export_platform, manifest_application_text += " \n"; return manifest_application_text; } + +Error _store_temp_file(const String &p_simplified_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, Vector &r_enc_data, EditorExportPlatform::SavedData &r_sd) { + Error err = OK; + Ref ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export", "tmp", false, &err); + if (err != OK) { + return err; + } + r_sd.path_utf8 = p_simplified_path.trim_prefix("res://").utf8(); + r_sd.ofs = 0; + r_sd.size = p_data.size(); + err = EditorExportPlatform::_encrypt_and_store_data(ftmp, p_simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, r_sd.encrypted); + if (err != OK) { + return err; + } + + r_enc_data.resize(ftmp->get_length()); + ftmp->seek(0); + ftmp->get_buffer(r_enc_data.ptrw(), r_enc_data.size()); + ftmp.unref(); + + // Store MD5 of original file. + { + unsigned char hash[16]; + CryptoCore::md5(p_data.ptr(), p_data.size(), hash); + r_sd.md5.resize(16); + for (int i = 0; i < 16; i++) { + r_sd.md5.write[i] = hash[i]; + } + } + return OK; +} diff --git a/platform/android/export/gradle_export_util.h b/platform/android/export/gradle_export_util.h index 045e7f14b8b..8ba70102c19 100644 --- a/platform/android/export/gradle_export_util.h +++ b/platform/android/export/gradle_export_util.h @@ -30,11 +30,13 @@ #pragma once +#include "core/crypto/crypto_core.h" #include "core/io/dir_access.h" #include "core/io/file_access.h" #include "core/io/zip_io.h" #include "core/os/os.h" #include "editor/export/editor_export.h" +#include "editor/export/editor_export_platform.h" const String GODOT_PROJECT_NAME_XML_STRING = R"( @@ -62,6 +64,7 @@ static const int XR_MODE_REGULAR = 0; static const int XR_MODE_OPENXR = 1; struct CustomExportData { + EditorExportPlatform::PackData pd; String assets_directory; String libs_directory; bool debug; @@ -81,6 +84,8 @@ int _get_app_category_value(int category_index); String _get_app_category_label(int category_index); +Error _store_temp_file(const String &p_simplified_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, Vector &r_enc_data, EditorExportPlatform::SavedData &r_sd); + // Utility method used to create a directory. Error create_directory(const String &p_dir);