1
0
mirror of https://github.com/godotengine/godot.git synced 2025-11-04 12:00:25 +00:00

Implement sparse bundle PCK support.

This commit is contained in:
Pāvels Nadtočajevs
2025-02-20 11:08:16 +02:00
committed by Rémi Verschelde
parent d89f4ab32f
commit 42733a2a5c
11 changed files with 351 additions and 146 deletions

View File

@@ -614,6 +614,8 @@ void ProjectSettings::_convert_to_last_version(int p_from_version) {
* appending '.pck' to the binary name (e.g. 'linux_game' -> 'linux_game.pck'). * appending '.pck' to the binary name (e.g. 'linux_game' -> 'linux_game.pck').
* o PCK with the same basename as the binary in the current working directory. * o PCK with the same basename as the binary in the current working directory.
* Same as above for the two possible PCK file names. * Same as above for the two possible PCK file names.
* - On Android, look for 'assets.sparsepck' and try loading it, if it doesn't work,
* proceed to the next step.
* - On relevant platforms (Android/iOS), lookup project file in OS resource path. * - On relevant platforms (Android/iOS), lookup project file in OS resource path.
* If found, load it or fail. * If found, load it or fail.
* - Lookup project file in passed `p_path` (--path passed by the user), i.e. we * - Lookup project file in passed `p_path` (--path passed by the user), i.e. we
@@ -701,6 +703,11 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
} }
} }
#ifdef ANDROID_ENABLED
// Attempt to load sparse PCK assets.
_load_resource_pack("res://assets.sparsepck", false, 0, true);
#endif
// Try to use the filesystem for files, according to OS. // Try to use the filesystem for files, according to OS.
// (Only Android -when reading from pck- and iOS use this.) // (Only Android -when reading from pck- and iOS use this.)

View File

@@ -159,7 +159,7 @@ Ref<FileAccess> FileAccess::open(const String &p_path, int p_mode_flags, Error *
//try packed data first //try packed data first
Ref<FileAccess> ret; Ref<FileAccess> ret;
if (!(p_mode_flags & WRITE) && PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled()) { if (!(p_mode_flags & WRITE) && !(p_mode_flags & SKIP_PACK) && PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled()) {
ret = PackedData::get_singleton()->try_open_path(p_path); ret = PackedData::get_singleton()->try_open_path(p_path);
if (ret.is_valid()) { if (ret.is_valid()) {
if (r_error) { if (r_error) {
@@ -170,7 +170,7 @@ Ref<FileAccess> FileAccess::open(const String &p_path, int p_mode_flags, Error *
} }
ret = create_for_path(p_path); ret = create_for_path(p_path);
Error err = ret->open_internal(p_path, p_mode_flags); Error err = ret->open_internal(p_path, p_mode_flags & ~SKIP_PACK);
if (r_error) { if (r_error) {
*r_error = err; *r_error = err;

View File

@@ -58,6 +58,7 @@ public:
WRITE = 2, WRITE = 2,
READ_WRITE = 3, READ_WRITE = 3,
WRITE_READ = 7, WRITE_READ = 7,
SKIP_PACK = 16,
}; };
enum UnixPermissionFlags : int32_t { enum UnixPermissionFlags : int32_t {

View File

@@ -45,7 +45,7 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t
return ERR_FILE_UNRECOGNIZED; return ERR_FILE_UNRECOGNIZED;
} }
void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted) { void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle) {
String simplified_path = p_path.simplify_path().trim_prefix("res://"); String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer()); PathMD5 pmd5(simplified_path.md5_buffer());
@@ -53,6 +53,7 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64
PackedFile pf; PackedFile pf;
pf.encrypted = p_encrypted; pf.encrypted = p_encrypted;
pf.bundle = p_bundle;
pf.pack = p_pkg_path; pf.pack = p_pkg_path;
pf.offset = p_ofs; pf.offset = p_ofs;
pf.size = p_size; pf.size = p_size;
@@ -268,6 +269,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
uint32_t pack_flags = f->get_32(); uint32_t pack_flags = f->get_32();
bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED); bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED);
bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3. bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3.
bool sparse_bundle = (pack_flags & PACK_SPARSE_BUNDLE);
uint64_t file_base = f->get_64(); uint64_t file_base = f->get_64();
if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) { if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) {
@@ -320,7 +322,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
if (flags & PACK_FILE_REMOVAL) { // The file was removed. if (flags & PACK_FILE_REMOVAL) { // The file was removed.
PackedData::get_singleton()->remove_path(path); PackedData::get_singleton()->remove_path(path);
} else { } else {
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED)); PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle);
} }
} }
@@ -360,7 +362,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f
for (const String &file_name : da->get_files()) { for (const String &file_name : da->get_files()) {
String file_path = p_path.path_join(file_name); String file_path = p_path.path_join(file_name);
uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false); PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false);
} }
for (const String &sub_dir_name : da->get_directories()) { for (const String &sub_dir_name : da->get_directories()) {
@@ -467,13 +469,19 @@ void FileAccessPack::close() {
f = Ref<FileAccess>(); f = Ref<FileAccess>();
} }
FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) : FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) {
pf(p_file), pf = p_file;
f(FileAccess::open(pf.pack, FileAccess::READ)) { if (pf.bundle) {
ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack))); String simplified_path = p_path.simplify_path();
f = FileAccess::open(simplified_path, FileAccess::READ | FileAccess::SKIP_PACK);
off = 0; // For the sparse pack offset is always zero.
} else {
f = FileAccess::open(pf.pack, FileAccess::READ);
f->seek(pf.offset);
off = pf.offset;
}
f->seek(pf.offset); ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack)));
off = pf.offset;
if (pf.encrypted) { if (pf.encrypted) {
Ref<FileAccessEncrypted> fae; Ref<FileAccessEncrypted> fae;

View File

@@ -48,6 +48,7 @@
enum PackFlags { enum PackFlags {
PACK_DIR_ENCRYPTED = 1 << 0, PACK_DIR_ENCRYPTED = 1 << 0,
PACK_REL_FILEBASE = 1 << 1, PACK_REL_FILEBASE = 1 << 1,
PACK_SPARSE_BUNDLE = 1 << 2,
}; };
enum PackFileFlags { enum PackFileFlags {
@@ -70,6 +71,7 @@ public:
uint8_t md5[16]; uint8_t md5[16];
PackSource *src = nullptr; PackSource *src = nullptr;
bool encrypted; bool encrypted;
bool bundle;
}; };
private: private:
@@ -114,7 +116,7 @@ private:
public: public:
void add_pack_source(PackSource *p_source); void add_pack_source(PackSource *p_source);
void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false); // for PackSource void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false); // for PackSource
void remove_path(const String &p_path); void remove_path(const String &p_path);
uint8_t *get_file_hash(const String &p_path); uint8_t *get_file_hash(const String &p_path);
HashSet<String> get_file_paths() const; HashSet<String> get_file_paths() const;

View File

@@ -252,41 +252,25 @@ void EditorExportPlatform::_unload_patches() {
PackedData::get_singleton()->clear(); PackedData::get_singleton()->clear();
} }
Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatform::_encrypt_and_store_data(Ref<FileAccess> p_fd, const String &p_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool &r_encrypt) {
ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export."); r_encrypt = false;
PackData *pd = (PackData *)p_userdata;
String simplified_path = p_path.simplify_path();
if (simplified_path.begins_with("uid://")) {
simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path();
print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path));
}
SavedData sd;
sd.path_utf8 = simplified_path.trim_prefix("res://").utf8();
sd.ofs = pd->f->get_position();
sd.size = p_data.size();
sd.encrypted = false;
for (int i = 0; i < p_enc_in_filters.size(); ++i) { for (int i = 0; i < p_enc_in_filters.size(); ++i) {
if (simplified_path.matchn(p_enc_in_filters[i]) || simplified_path.trim_prefix("res://").matchn(p_enc_in_filters[i])) { if (p_path.matchn(p_enc_in_filters[i]) || p_path.trim_prefix("res://").matchn(p_enc_in_filters[i])) {
sd.encrypted = true; r_encrypt = true;
break; break;
} }
} }
for (int i = 0; i < p_enc_ex_filters.size(); ++i) { for (int i = 0; i < p_enc_ex_filters.size(); ++i) {
if (simplified_path.matchn(p_enc_ex_filters[i]) || simplified_path.trim_prefix("res://").matchn(p_enc_ex_filters[i])) { if (p_path.matchn(p_enc_ex_filters[i]) || p_path.trim_prefix("res://").matchn(p_enc_ex_filters[i])) {
sd.encrypted = false; r_encrypt = false;
break; break;
} }
} }
Ref<FileAccessEncrypted> fae; Ref<FileAccessEncrypted> fae;
Ref<FileAccess> ftmp = pd->f; Ref<FileAccess> ftmp = p_fd;
if (r_encrypt) {
if (sd.encrypted) {
Vector<uint8_t> iv; Vector<uint8_t> iv;
if (p_seed != 0) { if (p_seed != 0) {
uint64_t seed = p_seed; uint64_t seed = p_seed;
@@ -319,12 +303,44 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
ftmp.unref(); ftmp.unref();
fae.unref(); fae.unref();
} }
return OK;
}
ERR_FAIL_COND_V(pd->f->get_position() - sd.ofs < (uint64_t)p_data.size(), ERR_FILE_CANT_WRITE); Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export.");
int pad = _get_pad(PCK_PADDING, pd->f->get_position()); PackData *pd = (PackData *)p_userdata;
for (int i = 0; i < pad; i++) {
pd->f->store_8(0); String simplified_path = p_path.simplify_path();
if (simplified_path.begins_with("uid://")) {
simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path();
print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path));
}
Ref<FileAccess> ftmp;
if (pd->use_sparse_pck) {
ftmp = FileAccess::open(pd->path.get_base_dir().path_join(simplified_path.trim_prefix("res://")), FileAccess::WRITE);
} else {
ftmp = pd->f;
}
SavedData sd;
sd.path_utf8 = simplified_path.trim_prefix("res://").utf8();
sd.ofs = (pd->use_sparse_pck) ? 0 : pd->f->get_position();
sd.size = p_data.size();
Error err = _encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted);
if (err != OK) {
return err;
}
if (!pd->use_sparse_pck) {
ERR_FAIL_COND_V(pd->f->get_position() - sd.ofs < (uint64_t)p_data.size(), ERR_FILE_CANT_WRITE);
}
if (!pd->use_sparse_pck) {
int pad = _get_pad(PCK_PADDING, pd->f->get_position());
for (int i = 0; i < pad; i++) {
pd->f->store_8(0);
}
} }
// Store MD5 of original file. // Store MD5 of original file.
@@ -1897,6 +1913,105 @@ Dictionary EditorExportPlatform::_save_zip_patch(const Ref<EditorExportPreset> &
return ret; return ret;
} }
bool EditorExportPlatform::_store_header(Ref<FileAccess> p_fd, bool p_enc, bool p_sparse, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs) {
p_fd->store_32(PACK_HEADER_MAGIC);
p_fd->store_32(PACK_FORMAT_VERSION);
p_fd->store_32(GODOT_VERSION_MAJOR);
p_fd->store_32(GODOT_VERSION_MINOR);
p_fd->store_32(GODOT_VERSION_PATCH);
uint32_t pack_flags = PACK_REL_FILEBASE;
if (p_enc) {
pack_flags |= PACK_DIR_ENCRYPTED;
}
if (p_sparse) {
pack_flags |= PACK_SPARSE_BUNDLE;
}
p_fd->store_32(pack_flags); // Flags.
r_file_base_ofs = p_fd->get_position();
p_fd->store_64(0); // Files base offset.
r_dir_base_ofs = p_fd->get_position();
p_fd->store_64(0); // Directory offset.
for (int i = 0; i < 16; i++) {
//reserved
p_fd->store_32(0);
}
return true;
}
bool EditorExportPlatform::_encrypt_and_store_directory(Ref<FileAccess> p_fd, PackData &p_pack_data, const Vector<uint8_t> &p_key, uint64_t p_seed, uint64_t p_file_base) {
Ref<FileAccessEncrypted> fae;
Ref<FileAccess> fhead = p_fd;
fhead->store_32(p_pack_data.file_ofs.size()); //amount of files
if (!p_key.is_empty()) {
uint64_t seed = p_seed;
fae.instantiate();
if (fae.is_null()) {
return false;
}
Vector<uint8_t> iv;
if (seed != 0) {
for (int i = 0; i < p_pack_data.file_ofs.size(); i++) {
for (int64_t j = 0; j < p_pack_data.file_ofs[i].path_utf8.length(); j++) {
seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].path_utf8.get_data()[j];
}
for (int64_t j = 0; j < p_pack_data.file_ofs[i].md5.size(); j++) {
seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].md5[j];
}
seed = ((seed << 5) + seed) ^ (p_pack_data.file_ofs[i].ofs - p_file_base);
seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].size;
}
RandomPCG rng = RandomPCG(seed);
iv.resize(16);
for (int i = 0; i < 16; i++) {
iv.write[i] = rng.rand() % 256;
}
}
Error err = fae->open_and_parse(fhead, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv);
if (err != OK) {
return false;
}
fhead = fae;
}
for (int i = 0; i < p_pack_data.file_ofs.size(); i++) {
uint32_t string_len = p_pack_data.file_ofs[i].path_utf8.length();
uint32_t pad = _get_pad(4, string_len);
fhead->store_32(string_len + pad);
fhead->store_buffer((const uint8_t *)p_pack_data.file_ofs[i].path_utf8.get_data(), string_len);
for (uint32_t j = 0; j < pad; j++) {
fhead->store_8(0);
}
fhead->store_64(p_pack_data.file_ofs[i].ofs - p_file_base);
fhead->store_64(p_pack_data.file_ofs[i].size); // pay attention here, this is where file is
fhead->store_buffer(p_pack_data.file_ofs[i].md5.ptr(), 16); //also save md5 for file
uint32_t flags = 0;
if (p_pack_data.file_ofs[i].encrypted) {
flags |= PACK_FILE_ENCRYPTED;
}
if (p_pack_data.file_ofs[i].removal) {
flags |= PACK_FILE_REMOVAL;
}
fhead->store_32(flags);
}
if (fae.is_valid()) {
fhead.unref();
fae.unref();
}
return true;
}
Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, Vector<SharedObject> *p_so_files, EditorExportSaveFunction p_save_func, EditorExportRemoveFunction p_remove_func, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) { Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, Vector<SharedObject> *p_so_files, EditorExportSaveFunction p_save_func, EditorExportRemoveFunction p_remove_func, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) {
EditorProgress ep("savepack", TTR("Packing"), 102, true); EditorProgress ep("savepack", TTR("Packing"), 102, true);
@@ -1940,31 +2055,10 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
} }
int64_t pck_start_pos = f->get_position(); int64_t pck_start_pos = f->get_position();
uint64_t file_base_ofs = 0;
uint64_t dir_base_ofs = 0;
// Write header. _store_header(f, p_preset->get_enc_pck() && p_preset->get_enc_directory(), false, file_base_ofs, dir_base_ofs);
f->store_32(PACK_HEADER_MAGIC);
f->store_32(PACK_FORMAT_VERSION);
f->store_32(GODOT_VERSION_MAJOR);
f->store_32(GODOT_VERSION_MINOR);
f->store_32(GODOT_VERSION_PATCH);
uint32_t pack_flags = PACK_REL_FILEBASE;
bool enc_pck = p_preset->get_enc_pck();
bool enc_directory = p_preset->get_enc_directory();
if (enc_pck && enc_directory) {
pack_flags |= PACK_DIR_ENCRYPTED;
}
f->store_32(pack_flags); // Flags.
uint64_t file_base_ofs = f->get_position();
f->store_64(0); // Files base.
uint64_t dir_base_ofs = f->get_position();
f->store_64(0); // Directory offset.
for (int i = 0; i < 16; i++) {
f->store_32(0); // Reserved.
}
// Align for first file. // Align for first file.
int file_padding = _get_pad(PCK_PADDING, f->get_position()); int file_padding = _get_pad(PCK_PADDING, f->get_position());
@@ -1982,6 +2076,7 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
pd.ep = &ep; pd.ep = &ep;
pd.f = f; pd.f = f;
pd.so_files = p_so_files; pd.so_files = p_so_files;
pd.path = p_path;
Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object); Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object);
@@ -2008,15 +2103,9 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
f->store_64(dir_offset - pck_start_pos); f->store_64(dir_offset - pck_start_pos);
f->seek(dir_offset); f->seek(dir_offset);
f->store_32(pd.file_ofs.size()); Vector<uint8_t> key;
if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) {
Ref<FileAccessEncrypted> fae;
Ref<FileAccess> fhead = f;
if (enc_pck && enc_directory) {
uint64_t seed = p_preset->get_seed();
String script_key = _get_script_encryption_key(p_preset); String script_key = _get_script_encryption_key(p_preset);
Vector<uint8_t> key;
key.resize(32); key.resize(32);
if (script_key.length() == 64) { if (script_key.length() == 64) {
for (int i = 0; i < 32; i++) { for (int i = 0; i < 32; i++) {
@@ -2043,67 +2132,11 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
key.write[i] = v; key.write[i] = v;
} }
} }
fae.instantiate();
if (fae.is_null()) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file."));
return ERR_CANT_CREATE;
}
Vector<uint8_t> iv;
if (seed != 0) {
for (int i = 0; i < pd.file_ofs.size(); i++) {
for (int64_t j = 0; j < pd.file_ofs[i].path_utf8.length(); j++) {
seed = ((seed << 5) + seed) ^ pd.file_ofs[i].path_utf8.get_data()[j];
}
for (int64_t j = 0; j < pd.file_ofs[i].md5.size(); j++) {
seed = ((seed << 5) + seed) ^ pd.file_ofs[i].md5[j];
}
seed = ((seed << 5) + seed) ^ (pd.file_ofs[i].ofs - file_base);
seed = ((seed << 5) + seed) ^ pd.file_ofs[i].size;
}
RandomPCG rng = RandomPCG(seed);
iv.resize(16);
for (int i = 0; i < 16; i++) {
iv.write[i] = rng.rand() % 256;
}
}
err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv);
if (err != OK) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't open encrypted file to write."));
return ERR_CANT_CREATE;
}
fhead = fae;
} }
for (int i = 0; i < pd.file_ofs.size(); i++) { if (!_encrypt_and_store_directory(f, pd, key, p_preset->get_seed(), file_base)) {
uint32_t string_len = pd.file_ofs[i].path_utf8.length(); add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file."));
uint32_t pad = _get_pad(4, string_len); return ERR_CANT_CREATE;
fhead->store_32(string_len + pad);
fhead->store_buffer((const uint8_t *)pd.file_ofs[i].path_utf8.get_data(), string_len);
for (uint32_t j = 0; j < pad; j++) {
fhead->store_8(0);
}
fhead->store_64(pd.file_ofs[i].ofs - file_base);
fhead->store_64(pd.file_ofs[i].size);
fhead->store_buffer(pd.file_ofs[i].md5.ptr(), 16);
uint32_t flags = 0;
if (pd.file_ofs[i].encrypted) {
flags |= PACK_FILE_ENCRYPTED;
}
if (pd.file_ofs[i].removal) {
flags |= PACK_FILE_REMOVAL;
}
fhead->store_32(flags);
}
if (fae.is_valid()) {
fhead.unref();
fae.unref();
} }
if (p_embed) { if (p_embed) {

View File

@@ -78,7 +78,6 @@ public:
String text; String text;
}; };
private:
struct SavedData { struct SavedData {
uint64_t ofs = 0; uint64_t ofs = 0;
uint64_t size = 0; uint64_t size = 0;
@@ -93,12 +92,20 @@ private:
}; };
struct PackData { struct PackData {
String path;
Ref<FileAccess> f; Ref<FileAccess> f;
Vector<SavedData> file_ofs; Vector<SavedData> file_ofs;
EditorProgress *ep = nullptr; EditorProgress *ep = nullptr;
Vector<SharedObject> *so_files = nullptr; Vector<SharedObject> *so_files = nullptr;
bool use_sparse_pck = false;
}; };
static bool _store_header(Ref<FileAccess> p_fd, bool p_enc, bool p_sparse, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs);
static bool _encrypt_and_store_directory(Ref<FileAccess> p_fd, PackData &p_pack_data, const Vector<uint8_t> &p_key, uint64_t p_seed, uint64_t p_file_base);
static Error _encrypt_and_store_data(Ref<FileAccess> p_fd, const String &p_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool &r_encrypt);
String _get_script_encryption_key(const Ref<EditorExportPreset> &p_preset) const;
private:
struct ZipData { struct ZipData {
void *zip = nullptr; void *zip = nullptr;
EditorProgress *ep = nullptr; EditorProgress *ep = nullptr;
@@ -151,7 +158,6 @@ private:
bool _is_editable_ancestor(Node *p_root, Node *p_node); bool _is_editable_ancestor(Node *p_root, Node *p_node);
String _export_customize(const String &p_path, LocalVector<Ref<EditorExportPlugin>> &customize_resources_plugins, LocalVector<Ref<EditorExportPlugin>> &customize_scenes_plugins, HashMap<String, FileExportCache> &export_cache, const String &export_base_path, bool p_force_save); String _export_customize(const String &p_path, LocalVector<Ref<EditorExportPlugin>> &customize_resources_plugins, LocalVector<Ref<EditorExportPlugin>> &customize_scenes_plugins, HashMap<String, FileExportCache> &export_cache, const String &export_base_path, bool p_force_save);
String _get_script_encryption_key(const Ref<EditorExportPreset> &p_preset) const;
protected: protected:
struct ExportNotifier { struct ExportNotifier {

View File

@@ -625,6 +625,7 @@ bool EditorExportPlatformAndroid::_should_compress_asset(const String &p_path, c
".cfb", // Don't let small config files slow-down startup ".cfb", // Don't let small config files slow-down startup
".scn", // Binary scenes are usually already compressed ".scn", // Binary scenes are usually already compressed
".ctex", // Streamable textures are usually already compressed ".ctex", // Streamable textures are usually already compressed
".pck", // Pack.
// Trailer for easier processing // Trailer for easier processing
nullptr nullptr
}; };
@@ -800,14 +801,25 @@ Error EditorExportPlatformAndroid::save_apk_so(void *p_userdata, const SharedObj
Error EditorExportPlatformAndroid::save_apk_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatformAndroid::save_apk_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
APKExportData *ed = static_cast<APKExportData *>(p_userdata); APKExportData *ed = static_cast<APKExportData *>(p_userdata);
String path = p_path.simplify_path(); String simplified_path = p_path.simplify_path();
if (path.begins_with("uid://")) { if (simplified_path.begins_with("uid://")) {
path = ResourceUID::uid_to_path(path).simplify_path(); simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path();
print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, path)); print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path));
} }
const String dst_path = path.replace_first("res://", "assets/");
store_in_apk(ed, dst_path, p_data, _should_compress_asset(path, p_data) ? Z_DEFLATED : 0); Vector<uint8_t> enc_data;
EditorExportPlatform::SavedData sd;
Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, enc_data, sd);
if (err != OK) {
return err;
}
const String dst_path = String("assets/") + simplified_path.trim_prefix("res://");
print_verbose("Saving project files from " + simplified_path + " into " + dst_path);
store_in_apk(ed, dst_path, enc_data, _should_compress_asset(simplified_path, enc_data) ? Z_DEFLATED : 0);
ed->pd.file_ofs.push_back(sd);
return OK; return OK;
} }
@@ -3409,6 +3421,68 @@ Error EditorExportPlatformAndroid::export_project(const Ref<EditorExportPreset>
return export_project_helper(p_preset, p_debug, p_path, export_format, should_sign, p_flags); return export_project_helper(p_preset, p_debug, p_path, export_format, should_sign, p_flags);
} }
Error EditorExportPlatformAndroid::_generate_sparse_pck_metadata(const Ref<EditorExportPreset> &p_preset, PackData &p_pack_data, Vector<uint8_t> &r_data) {
Error err;
Ref<FileAccess> ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export_index", "tmp", false, &err);
if (err != OK) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not create temporary file!"));
return err;
}
int64_t pck_start_pos = ftmp->get_position();
uint64_t file_base_ofs = 0;
uint64_t dir_base_ofs = 0;
EditorExportPlatform::_store_header(ftmp, p_preset->get_enc_pck() && p_preset->get_enc_directory(), true, file_base_ofs, dir_base_ofs);
// Write directory.
uint64_t dir_offset = ftmp->get_position();
ftmp->seek(dir_base_ofs);
ftmp->store_64(dir_offset - pck_start_pos);
ftmp->seek(dir_offset);
Vector<uint8_t> key;
if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) {
String script_key = _get_script_encryption_key(p_preset);
key.resize(32);
if (script_key.length() == 64) {
for (int i = 0; i < 32; i++) {
int v = 0;
if (i * 2 < script_key.length()) {
char32_t ct = script_key[i * 2];
if (is_digit(ct)) {
ct = ct - '0';
} else if (ct >= 'a' && ct <= 'f') {
ct = 10 + ct - 'a';
}
v |= ct << 4;
}
if (i * 2 + 1 < script_key.length()) {
char32_t ct = script_key[i * 2 + 1];
if (is_digit(ct)) {
ct = ct - '0';
} else if (ct >= 'a' && ct <= 'f') {
ct = 10 + ct - 'a';
}
v |= ct;
}
key.write[i] = v;
}
}
}
if (!EditorExportPlatform::_encrypt_and_store_directory(ftmp, p_pack_data, key, p_preset->get_seed(), 0)) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file."));
return ERR_CANT_CREATE;
}
r_data.resize(ftmp->get_length());
ftmp->seek(0);
ftmp->get_buffer(r_data.ptrw(), r_data.size());
ftmp.unref();
return OK;
}
Error EditorExportPlatformAndroid::export_project_helper(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int export_format, bool should_sign, BitField<EditorExportPlatform::DebugFlags> p_flags) { Error EditorExportPlatformAndroid::export_project_helper(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int export_format, bool should_sign, BitField<EditorExportPlatform::DebugFlags> p_flags) {
ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags); ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags);
@@ -3536,7 +3610,22 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref<EditorExportP
if (p_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT)) { if (p_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT)) {
err = export_project_files(p_preset, p_debug, ignore_apk_file, nullptr, &user_data, copy_gradle_so); err = export_project_files(p_preset, p_debug, ignore_apk_file, nullptr, &user_data, copy_gradle_so);
} else { } else {
user_data.pd.path = "assets.sparsepck";
user_data.pd.use_sparse_pck = true;
err = export_project_files(p_preset, p_debug, rename_and_store_file_in_gradle_project, nullptr, &user_data, copy_gradle_so); err = export_project_files(p_preset, p_debug, rename_and_store_file_in_gradle_project, nullptr, &user_data, copy_gradle_so);
Vector<uint8_t> enc_data;
err = _generate_sparse_pck_metadata(p_preset, user_data.pd, enc_data);
if (err != OK) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!"));
return err;
}
err = store_file_at_path(user_data.assets_directory + "/assets.sparsepck", enc_data);
if (err != OK) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not write PCK directory!"));
return err;
}
} }
if (err != OK) { if (err != OK) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not export project files to gradle project.")); add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not export project files to gradle project."));
@@ -3988,7 +4077,18 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref<EditorExportP
APKExportData ed; APKExportData ed;
ed.ep = &ep; ed.ep = &ep;
ed.apk = unaligned_apk; ed.apk = unaligned_apk;
ed.pd.path = "assets.sparsepck";
ed.pd.use_sparse_pck = true;
err = export_project_files(p_preset, p_debug, save_apk_file, nullptr, &ed, save_apk_so); err = export_project_files(p_preset, p_debug, save_apk_file, nullptr, &ed, save_apk_so);
Vector<uint8_t> enc_data;
err = _generate_sparse_pck_metadata(p_preset, ed.pd, enc_data);
if (err != OK) {
add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!"));
return err;
}
store_in_apk(&ed, "assets/assets.sparsepck", enc_data, 0);
} }
} }

View File

@@ -73,6 +73,7 @@ class EditorExportPlatformAndroid : public EditorExportPlatform {
}; };
struct APKExportData { struct APKExportData {
EditorExportPlatform::PackData pd;
zipFile apk; zipFile apk;
EditorProgress *ep = nullptr; EditorProgress *ep = nullptr;
}; };
@@ -193,6 +194,8 @@ class EditorExportPlatformAndroid : public EditorExportPlatform {
bool _uses_vulkan(const Ref<EditorExportPreset> &p_preset) const; bool _uses_vulkan(const Ref<EditorExportPreset> &p_preset) const;
Error _generate_sparse_pck_metadata(const Ref<EditorExportPreset> &p_preset, PackData &p_pack_data, Vector<uint8_t> &r_data);
protected: protected:
void _notification(int p_what); void _notification(int p_what);

View File

@@ -172,15 +172,24 @@ Error store_string_at_path(const String &p_path, const String &p_data) {
Error rename_and_store_file_in_gradle_project(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error rename_and_store_file_in_gradle_project(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
CustomExportData *export_data = static_cast<CustomExportData *>(p_userdata); CustomExportData *export_data = static_cast<CustomExportData *>(p_userdata);
String path = p_path.simplify_path(); String simplified_path = p_path.simplify_path();
if (path.begins_with("uid://")) { if (simplified_path.begins_with("uid://")) {
path = ResourceUID::uid_to_path(path).simplify_path(); simplified_path = ResourceUID::uid_to_path(simplified_path).simplify_path();
print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, path)); print_verbose(vformat(R"(UID referenced exported file name "%s" was replaced with "%s".)", p_path, simplified_path));
} }
const String dst_path = path.replace_first("res://", export_data->assets_directory + "/");
print_verbose("Saving project files from " + path + " into " + dst_path); Vector<uint8_t> enc_data;
Error err = store_file_at_path(dst_path, p_data); EditorExportPlatform::SavedData sd;
Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, enc_data, sd);
if (err != OK) {
return err;
}
const String dst_path = export_data->assets_directory + String("/") + simplified_path.trim_prefix("res://");
print_verbose("Saving project files from " + simplified_path + " into " + dst_path);
err = store_file_at_path(dst_path, enc_data);
export_data->pd.file_ofs.push_back(sd);
return err; return err;
} }
@@ -354,3 +363,34 @@ String _get_application_tag(const Ref<EditorExportPlatform> &p_export_platform,
manifest_application_text += " </application>\n"; manifest_application_text += " </application>\n";
return manifest_application_text; return manifest_application_text;
} }
Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, Vector<uint8_t> &r_enc_data, EditorExportPlatform::SavedData &r_sd) {
Error err = OK;
Ref<FileAccess> ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export", "tmp", false, &err);
if (err != OK) {
return err;
}
r_sd.path_utf8 = p_simplified_path.trim_prefix("res://").utf8();
r_sd.ofs = 0;
r_sd.size = p_data.size();
err = EditorExportPlatform::_encrypt_and_store_data(ftmp, p_simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, r_sd.encrypted);
if (err != OK) {
return err;
}
r_enc_data.resize(ftmp->get_length());
ftmp->seek(0);
ftmp->get_buffer(r_enc_data.ptrw(), r_enc_data.size());
ftmp.unref();
// Store MD5 of original file.
{
unsigned char hash[16];
CryptoCore::md5(p_data.ptr(), p_data.size(), hash);
r_sd.md5.resize(16);
for (int i = 0; i < 16; i++) {
r_sd.md5.write[i] = hash[i];
}
}
return OK;
}

View File

@@ -30,11 +30,13 @@
#pragma once #pragma once
#include "core/crypto/crypto_core.h"
#include "core/io/dir_access.h" #include "core/io/dir_access.h"
#include "core/io/file_access.h" #include "core/io/file_access.h"
#include "core/io/zip_io.h" #include "core/io/zip_io.h"
#include "core/os/os.h" #include "core/os/os.h"
#include "editor/export/editor_export.h" #include "editor/export/editor_export.h"
#include "editor/export/editor_export_platform.h"
const String GODOT_PROJECT_NAME_XML_STRING = R"(<?xml version="1.0" encoding="utf-8"?> const String GODOT_PROJECT_NAME_XML_STRING = R"(<?xml version="1.0" encoding="utf-8"?>
<!--WARNING: THIS FILE WILL BE OVERWRITTEN AT BUILD TIME--> <!--WARNING: THIS FILE WILL BE OVERWRITTEN AT BUILD TIME-->
@@ -62,6 +64,7 @@ static const int XR_MODE_REGULAR = 0;
static const int XR_MODE_OPENXR = 1; static const int XR_MODE_OPENXR = 1;
struct CustomExportData { struct CustomExportData {
EditorExportPlatform::PackData pd;
String assets_directory; String assets_directory;
String libs_directory; String libs_directory;
bool debug; bool debug;
@@ -81,6 +84,8 @@ int _get_app_category_value(int category_index);
String _get_app_category_label(int category_index); String _get_app_category_label(int category_index);
Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, Vector<uint8_t> &r_enc_data, EditorExportPlatform::SavedData &r_sd);
// Utility method used to create a directory. // Utility method used to create a directory.
Error create_directory(const String &p_dir); Error create_directory(const String &p_dir);