godotengine/godot
1
0
Fork 0
mirror of https://github.com/godotengine/godot.git synced 2025-12-02 16:48:55 +00:00
Code Releases Wiki Activity

Add support for delta encoding to patch PCKs

Browse Source
This commit is contained in:
Mikael Hermansson
2025-10-24 16:10:45 +02:00
parent 9dd6c4dbac
commit 0cc88f34da
17 changed files with 851 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
core/io/delta_encoding.cpp Normal file
View File

@@ -0,0 +1,110 @@
/**************************************************************************/
/* delta_encoding.cpp */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#include "delta_encoding.h"
#include <zstd.h>
#define ERR_FAIL_ZSTD_V_MSG(m_result, m_retval, m_msg) \
ERR_FAIL_COND_V_MSG(ZSTD_isError(m_result), m_retval, vformat("%s Zstandard reported error code %d: \"%s\".", m_msg, ZSTD_getErrorCode(m_result), ZSTD_getErrorString(ZSTD_getErrorCode(m_result))))
struct ZstdCompressionContext {
ZSTD_CCtx *context = ZSTD_createCCtx();
~ZstdCompressionContext() { ZSTD_freeCCtx(context); }
operator ZSTD_CCtx *() { return context; }
};
struct ZstdDecompressionContext {
ZSTD_DCtx *context = ZSTD_createDCtx();
~ZstdDecompressionContext() { ZSTD_freeDCtx(context); }
operator ZSTD_DCtx *() { return context; }
};
static constexpr uint8_t DELTA_MAGIC[4] = { 'G', 'D', 'D', 'L' };
static constexpr uint8_t DELTA_VERSION_NUMBER = 1;
static constexpr size_t DELTA_HEADER_SIZE = 5;
Error DeltaEncoding::encode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_new_data, Vector<uint8_t> &r_delta, int p_compression_level) {
size_t zstd_result = ZSTD_compressBound(p_new_data.size());
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Calculating compression bounds failed.");
r_delta.reserve_exact(DELTA_HEADER_SIZE + zstd_result);
r_delta.resize(DELTA_HEADER_SIZE + zstd_result);
memcpy(r_delta.ptrw(), DELTA_MAGIC, 4);
r_delta.write[4] = DELTA_VERSION_NUMBER;
ZstdCompressionContext zstd_context;
ZSTD_parameters zstd_params = ZSTD_getParams(p_compression_level, p_new_data.size(), p_old_data.size());
zstd_params.fParams.contentSizeFlag = 1;
zstd_params.fParams.checksumFlag = 1;
zstd_result = ZSTD_CCtx_setParams(zstd_context, zstd_params);
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Setting compression parameters failed.");
zstd_result = ZSTD_CCtx_refPrefix(zstd_context, p_old_data.ptr(), p_old_data.size());
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Setting prefix dictionary failed.");
zstd_result = ZSTD_compress2(zstd_context, r_delta.ptrw() + DELTA_HEADER_SIZE, r_delta.size() - DELTA_HEADER_SIZE, p_new_data.ptr(), p_new_data.size());
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Compression failed.");
r_delta.resize(DELTA_HEADER_SIZE + zstd_result);
return OK;
}
Error DeltaEncoding::decode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_delta, Vector<uint8_t> &r_new_data) {
ERR_FAIL_COND_V_MSG(p_delta.size() < DELTA_HEADER_SIZE, ERR_INVALID_DATA, vformat("Failed to decode delta. File size (%d) is too small.", p_delta.size()));
uint8_t magic[4];
memcpy(magic, p_delta.ptr(), sizeof(magic));
uint8_t version = p_delta[4];
ERR_FAIL_COND_V_MSG(memcmp(magic, DELTA_MAGIC, 4) != 0, ERR_FILE_CORRUPT, "Failed to decode delta. Header is invalid.");
ERR_FAIL_COND_V_MSG(version != DELTA_VERSION_NUMBER, ERR_FILE_UNRECOGNIZED, vformat("Failed to decode delta. Expected version %d but found %d.", DELTA_VERSION_NUMBER, version));
size_t zstd_result = ZSTD_findDecompressedSize(p_delta.ptr() + DELTA_HEADER_SIZE, p_delta.size() - DELTA_HEADER_SIZE);
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to decode delta. Unable to find decompressed size.");
r_new_data.reserve_exact(zstd_result);
r_new_data.resize(zstd_result);
ZstdDecompressionContext zstd_context;
zstd_result = ZSTD_DCtx_refPrefix(zstd_context, p_old_data.ptr(), p_old_data.size());
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to decode delta. Setting prefix dictionary failed.");
zstd_result = ZSTD_decompressDCtx(zstd_context, r_new_data.ptrw(), r_new_data.size(), p_delta.ptr() + DELTA_HEADER_SIZE, p_delta.size() - DELTA_HEADER_SIZE);
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to decode delta. Decompression failed.");
ERR_FAIL_COND_V(zstd_result != (size_t)r_new_data.size(), ERR_FILE_CORRUPT);
return OK;
}

39
core/io/delta_encoding.h Normal file
View File

@@ -0,0 +1,39 @@
/**************************************************************************/
/* delta_encoding.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#pragma once
#include "core/io/file_access.h"
class DeltaEncoding {
public:
static Error encode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_new_data, Vector<uint8_t> &r_delta, int p_compression_level = 19);
static Error decode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_delta, Vector<uint8_t> &r_new_data);
};

49
core/io/file_access_pack.cpp
View File

@@ -31,6 +31,7 @@
#include "file_access_pack.h" #include "file_access_pack.h"
#include "core/io/file_access_encrypted.h" #include "core/io/file_access_encrypted.h"
#include "core/io/file_access_patched.h"
#include "core/object/script_language.h" #include "core/object/script_language.h"
#include "core/os/os.h" #include "core/os/os.h"
#include "core/version.h" #include "core/version.h"
@@ -45,7 +46,7 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t
return ERR_FILE_UNRECOGNIZED; return ERR_FILE_UNRECOGNIZED;
} }
void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle) { void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle, bool p_delta) {
String simplified_path = p_path.simplify_path().trim_prefix("res://"); String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer()); PathMD5 pmd5(simplified_path.md5_buffer());
@@ -54,6 +55,7 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64
PackedFile pf; PackedFile pf;
pf.encrypted = p_encrypted; pf.encrypted = p_encrypted;
pf.bundle = p_bundle; pf.bundle = p_bundle;
pf.delta = p_delta;
pf.pack = p_pkg_path; pf.pack = p_pkg_path;
pf.offset = p_ofs; pf.offset = p_ofs;
pf.size = p_size; pf.size = p_size;
@@ -62,8 +64,11 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64
} }
pf.src = p_src; pf.src = p_src;
if (!exists || p_replace_files) { if (p_delta) {
delta_patches[pmd5].push_back(pf);
} else if (!exists || p_replace_files) {
files[pmd5] = pf; files[pmd5] = pf;
delta_patches[pmd5].clear();
} }
if (!exists) { if (!exists) {
@@ -137,6 +142,28 @@ uint8_t *PackedData::get_file_hash(const String &p_path) {
return E->value.md5; return E->value.md5;
} }
Vector<PackedData::PackedFile> PackedData::get_delta_patches(const String &p_path) const {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());
HashMap<PathMD5, Vector<PackedFile>, PathMD5>::ConstIterator E = delta_patches.find(pmd5);
if (!E) {
return Vector<PackedFile>();
}
return E->value;
}
bool PackedData::has_delta_patches(const String &p_path) const {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());
HashMap<PathMD5, Vector<PackedFile>, PathMD5>::ConstIterator E = delta_patches.find(pmd5);
if (!E) {
return false;
}
return !E->value.is_empty();
}
HashSet<String> PackedData::get_file_paths() const { HashSet<String> PackedData::get_file_paths() const {
HashSet<String> file_paths; HashSet<String> file_paths;
_get_file_paths(root, root->name, file_paths); _get_file_paths(root, root->name, file_paths);
@@ -155,6 +182,7 @@ void PackedData::_get_file_paths(PackedDir *p_dir, const String &p_parent_dir, H
void PackedData::clear() { void PackedData::clear() {
files.clear(); files.clear();
delta_patches.clear();
_free_packed_dirs(root); _free_packed_dirs(root);
root = memnew(PackedDir); root = memnew(PackedDir);
} }
@@ -322,7 +350,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
if (flags & PACK_FILE_REMOVAL) { // The file was removed. if (flags & PACK_FILE_REMOVAL) { // The file was removed.
PackedData::get_singleton()->remove_path(path); PackedData::get_singleton()->remove_path(path);
} else { } else {
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle); PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle, (flags & PACK_FILE_DELTA));
} }
} }
@@ -330,7 +358,17 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
} }
Ref<FileAccess> PackedSourcePCK::get_file(const String &p_path, PackedData::PackedFile *p_file) { Ref<FileAccess> PackedSourcePCK::get_file(const String &p_path, PackedData::PackedFile *p_file) {
return memnew(FileAccessPack(p_path, *p_file)); Ref<FileAccess> file(memnew(FileAccessPack(p_path, *p_file)));
if (PackedData::get_singleton()->has_delta_patches(p_path)) {
Ref<FileAccessPatched> file_patched;
file_patched.instantiate();
Error err = file_patched->open_custom(file);
ERR_FAIL_COND_V(err != OK, Ref<FileAccess>());
file = file_patched;
}
return file;
} }
////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////
@@ -362,7 +400,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f
for (const String &file_name : da->get_files()) { for (const String &file_name : da->get_files()) {
String file_path = p_path.path_join(file_name); String file_path = p_path.path_join(file_name);
uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false); PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false, false);
} }
for (const String &sub_dir_name : da->get_directories()) { for (const String &sub_dir_name : da->get_directories()) {
@@ -470,6 +508,7 @@ void FileAccessPack::close() {
} }
FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) { FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) {
path = p_path;
pf = p_file; pf = p_file;
if (pf.bundle) { if (pf.bundle) {
String simplified_path = p_path.simplify_path(); String simplified_path = p_path.simplify_path();

11
core/io/file_access_pack.h
View File

@@ -54,6 +54,7 @@ enum PackFlags {
enum PackFileFlags { enum PackFileFlags {
PACK_FILE_ENCRYPTED = 1 << 0, PACK_FILE_ENCRYPTED = 1 << 0,
PACK_FILE_REMOVAL = 1 << 1, PACK_FILE_REMOVAL = 1 << 1,
PACK_FILE_DELTA = 1 << 2,
}; };
class PackSource; class PackSource;
@@ -72,6 +73,7 @@ public:
PackSource *src = nullptr; PackSource *src = nullptr;
bool encrypted; bool encrypted;
bool bundle; bool bundle;
bool delta;
}; };
private: private:
@@ -103,6 +105,7 @@ private:
}; };
HashMap<PathMD5, PackedFile, PathMD5> files; HashMap<PathMD5, PackedFile, PathMD5> files;
HashMap<PathMD5, Vector<PackedFile>, PathMD5> delta_patches;
Vector<PackSource *> sources; Vector<PackSource *> sources;
@@ -116,9 +119,11 @@ private:
public: public:
void add_pack_source(PackSource *p_source); void add_pack_source(PackSource *p_source);
void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false); // for PackSource void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false, bool p_delta = false); // for PackSource
void remove_path(const String &p_path); void remove_path(const String &p_path);
uint8_t *get_file_hash(const String &p_path); uint8_t *get_file_hash(const String &p_path);
Vector<PackedFile> get_delta_patches(const String &p_path) const;
bool has_delta_patches(const String &p_path) const;
HashSet<String> get_file_paths() const; HashSet<String> get_file_paths() const;
void set_disabled(bool p_disabled) { disabled = p_disabled; } void set_disabled(bool p_disabled) { disabled = p_disabled; }
@@ -166,6 +171,7 @@ class FileAccessPack : public FileAccess {
GDSOFTCLASS(FileAccessPack, FileAccess); GDSOFTCLASS(FileAccessPack, FileAccess);
PackedData::PackedFile pf; PackedData::PackedFile pf;
String path;
mutable uint64_t pos; mutable uint64_t pos;
mutable bool eof; mutable bool eof;
uint64_t off; uint64_t off;
@@ -186,6 +192,9 @@ class FileAccessPack : public FileAccess {
public: public:
virtual bool is_open() const override; virtual bool is_open() const override;
virtual String get_path() const override { return path; }
virtual String get_path_absolute() const override { return path; }
virtual void seek(uint64_t p_position) override; virtual void seek(uint64_t p_position) override;
virtual void seek_end(int64_t p_position = 0) override; virtual void seek_end(int64_t p_position = 0) override;
virtual uint64_t get_position() const override; virtual uint64_t get_position() const override;

203
core/io/file_access_patched.cpp Normal file
View File

@@ -0,0 +1,203 @@
/**************************************************************************/
/* file_access_patched.cpp */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#include "file_access_patched.h"
#include "file_access_pack.h"
#include "core/io/delta_encoding.h"
#include "core/os/os.h"
Error FileAccessPatched::_apply_patch() const {
ERR_FAIL_COND_V(!is_open(), FAILED);
String path = old_file->get_path();
Vector<PackedData::PackedFile> delta_patches = PackedData::get_singleton()->get_delta_patches(path);
Vector<uint8_t> old_file_data = old_file->get_buffer(old_file->get_length());
for (int i = 0; i < delta_patches.size(); ++i) {
const PackedData::PackedFile &delta_patch = delta_patches[i];
ERR_FAIL_COND_V(delta_patch.bundle, FAILED);
Error err = OK;
uint64_t total_usec_start = OS::get_singleton()->get_ticks_usec();
uint64_t io_usec_start = OS::get_singleton()->get_ticks_usec();
Ref<FileAccess> patch_file = FileAccess::open(delta_patch.pack, FileAccess::READ, &err);
ERR_FAIL_COND_V(err != OK, err);
patch_file->seek(delta_patch.offset);
ERR_FAIL_COND_V(patch_file->get_error() != OK, patch_file->get_error());
Vector<uint8_t> patch_data = patch_file->get_buffer(delta_patch.size);
ERR_FAIL_COND_V(patch_data.is_empty(), ERR_FILE_CANT_READ);
uint64_t io_usec_end = OS::get_singleton()->get_ticks_usec();
uint64_t decode_usec_start = OS::get_singleton()->get_ticks_usec();
Vector<uint8_t> new_file_data;
err = DeltaEncoding::decode_delta(old_file_data, patch_data, new_file_data);
ERR_FAIL_COND_V_MSG(err != OK, err, vformat("Failed to apply delta patch (%d of %d) to \"%s\".", i + 1, delta_patches.size(), path));
uint64_t decode_usec_end = OS::get_singleton()->get_ticks_usec();
old_file_data = new_file_data;
uint64_t total_usec_end = OS::get_singleton()->get_ticks_usec();
print_verbose(vformat(U"Applied delta patch to \"%s\" from \"%s\" in %d μs (%d μs I/O, %d μs decoding).", path, delta_patch.pack.get_file(), total_usec_end - total_usec_start, io_usec_end - io_usec_start, decode_usec_end - decode_usec_start));
}
patched_file_data = old_file_data;
patched_file.instantiate();
return patched_file->open_custom(patched_file_data.ptr(), patched_file_data.size());
}
bool FileAccessPatched::_try_apply_patch() const {
if (last_error != OK) {
return false;
}
if (patched_file.is_valid()) {
return true;
}
last_error = _apply_patch();
return last_error == OK;
}
Error FileAccessPatched::open_custom(const Ref<FileAccess> &p_old_file) {
close();
if (!p_old_file->is_open()) {
last_error = ERR_FILE_CANT_OPEN;
return last_error;
}
old_file = p_old_file;
return OK;
}
bool FileAccessPatched::is_open() const {
return old_file.is_valid() && old_file->is_open();
}
void FileAccessPatched::seek(uint64_t p_position) {
if (!_try_apply_patch()) {
return;
}
patched_file->seek(p_position);
}
void FileAccessPatched::seek_end(int64_t p_position) {
if (!_try_apply_patch()) {
return;
}
patched_file->seek_end(p_position);
}
uint64_t FileAccessPatched::get_position() const {
if (!_try_apply_patch()) {
return 0;
}
return patched_file->get_position();
}
uint64_t FileAccessPatched::get_length() const {
if (!_try_apply_patch()) {
return 0;
}
return patched_file->get_length();
}
bool FileAccessPatched::eof_reached() const {
if (!_try_apply_patch()) {
return true;
}
return patched_file->eof_reached();
}
Error FileAccessPatched::get_error() const {
if (last_error != OK) {
return last_error;
}
if (patched_file.is_valid()) {
Error inner_error = patched_file->get_error();
if (inner_error != OK) {
return inner_error;
}
}
return last_error;
}
bool FileAccessPatched::store_buffer(const uint8_t *p_src, uint64_t p_length) {
if (!_try_apply_patch()) {
return false;
}
return patched_file->store_buffer(p_src, p_length);
}
uint64_t FileAccessPatched::get_buffer(uint8_t *p_dst, uint64_t p_length) const {
if (!_try_apply_patch()) {
return 0;
}
return patched_file->get_buffer(p_dst, p_length);
}
void FileAccessPatched::flush() {
if (!_try_apply_patch()) {
return;
}
patched_file->flush();
}
void FileAccessPatched::close() {
old_file = Ref<FileAccess>();
patched_file = Ref<FileAccessMemory>();
patched_file_data.clear();
last_error = OK;
}
bool FileAccessPatched::file_exists(const String &p_name) {
ERR_FAIL_COND_V(old_file.is_null(), false);
return old_file->file_exists(p_name);
}

84
core/io/file_access_patched.h Normal file
View File

@@ -0,0 +1,84 @@
/**************************************************************************/
/* file_access_patched.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#pragma once
#include "file_access.h"
#include "file_access_memory.h"
class FileAccessPatched : public FileAccess {
GDSOFTCLASS(FileAccessPatched, FileAccess);
Ref<FileAccess> old_file;
mutable Vector<uint8_t> patched_file_data;
mutable Ref<FileAccessMemory> patched_file;
mutable Error last_error = OK;
Error _apply_patch() const;
bool _try_apply_patch() const;
protected:
virtual BitField<UnixPermissionFlags> _get_unix_permissions(const String &p_file) override { return 0; }
virtual Error _set_unix_permissions(const String &p_file, BitField<UnixPermissionFlags> p_permissions) override { return FAILED; }
virtual bool _get_hidden_attribute(const String &p_file) override { return false; }
virtual Error _set_hidden_attribute(const String &p_file, bool p_hidden) override { return ERR_UNAVAILABLE; }
virtual bool _get_read_only_attribute(const String &p_file) override { return false; }
virtual Error _set_read_only_attribute(const String &p_file, bool p_ro) override { return ERR_UNAVAILABLE; }
virtual uint64_t _get_modified_time(const String &p_file) override { return 0; }
virtual uint64_t _get_access_time(const String &p_file) override { return 0; }
virtual int64_t _get_size(const String &p_file) override { return -1; }
virtual Error open_internal(const String &p_path, int p_mode_flags) override { return ERR_UNAVAILABLE; }
public:
Error open_custom(const Ref<FileAccess> &p_old_file);
virtual bool is_open() const override;
virtual void seek(uint64_t p_position) override;
virtual void seek_end(int64_t p_position = 0) override;
virtual uint64_t get_position() const override;
virtual uint64_t get_length() const override;
virtual bool eof_reached() const override;
virtual Error get_error() const override;
virtual bool store_buffer(const uint8_t *p_src, uint64_t p_length) override;
virtual uint64_t get_buffer(uint8_t *p_dst, uint64_t p_length) const override;
virtual Error resize(int64_t p_length) override { return ERR_UNAVAILABLE; }
virtual void flush() override;
virtual void close() override;
virtual bool file_exists(const String &p_name) override;
};

22
editor/export/editor_export.cpp
View File

@@ -83,7 +83,13 @@ void EditorExport::_save() {
config->set_value(section, "include_filter", preset->get_include_filter()); config->set_value(section, "include_filter", preset->get_include_filter());
config->set_value(section, "exclude_filter", preset->get_exclude_filter()); config->set_value(section, "exclude_filter", preset->get_exclude_filter());
config->set_value(section, "export_path", preset->get_export_path()); config->set_value(section, "export_path", preset->get_export_path());
config->set_value(section, "patches", preset->get_patches()); config->set_value(section, "patches", preset->get_patches());
config->set_value(section, "patch_delta_encoding", preset->is_patch_delta_encoding_enabled());
config->set_value(section, "patch_delta_compression_level_zstd", preset->get_patch_delta_zstd_level());
config->set_value(section, "patch_delta_min_reduction", preset->get_patch_delta_min_reduction());
config->set_value(section, "patch_delta_include_filters", preset->get_patch_delta_include_filter());
config->set_value(section, "patch_delta_exclude_filters", preset->get_patch_delta_exclude_filter());
config->set_value(section, "encryption_include_filters", preset->get_enc_in_filter()); config->set_value(section, "encryption_include_filters", preset->get_enc_in_filter());
config->set_value(section, "encryption_exclude_filters", preset->get_enc_ex_filter()); config->set_value(section, "encryption_exclude_filters", preset->get_enc_ex_filter());
@@ -332,6 +338,22 @@ void EditorExport::load_config() {
preset->set_script_export_mode(config->get_value(section, "script_export_mode", EditorExportPreset::MODE_SCRIPT_BINARY_TOKENS_COMPRESSED)); preset->set_script_export_mode(config->get_value(section, "script_export_mode", EditorExportPreset::MODE_SCRIPT_BINARY_TOKENS_COMPRESSED));
preset->set_patches(config->get_value(section, "patches", Vector<String>())); preset->set_patches(config->get_value(section, "patches", Vector<String>()));
if (config->has_section_key(section, "patch_delta_encoding")) {
preset->set_patch_delta_encoding_enabled(config->get_value(section, "patch_delta_encoding"));
}
if (config->has_section_key(section, "patch_delta_compression_level_zstd")) {
preset->set_patch_delta_zstd_level(config->get_value(section, "patch_delta_compression_level_zstd"));
}
if (config->has_section_key(section, "patch_delta_min_reduction")) {
preset->set_patch_delta_min_reduction(config->get_value(section, "patch_delta_min_reduction"));
}
if (config->has_section_key(section, "patch_delta_include_filters")) {
preset->set_patch_delta_include_filter(config->get_value(section, "patch_delta_include_filters"));
}
if (config->has_section_key(section, "patch_delta_exclude_filters")) {
preset->set_patch_delta_exclude_filter(config->get_value(section, "patch_delta_exclude_filters"));
}
if (config->has_section_key(section, "seed")) { if (config->has_section_key(section, "seed")) {
preset->set_seed(config->get_value(section, "seed")); preset->set_seed(config->get_value(section, "seed"));
} }

152
editor/export/editor_export_platform.cpp
View File

@@ -35,6 +35,7 @@
#include "core/config/project_settings.h" #include "core/config/project_settings.h"
#include "core/crypto/crypto_core.h" #include "core/crypto/crypto_core.h"
#include "core/extension/gdextension.h" #include "core/extension/gdextension.h"
#include "core/io/delta_encoding.h"
#include "core/io/dir_access.h" #include "core/io/dir_access.h"
#include "core/io/file_access_encrypted.h" #include "core/io/file_access_encrypted.h"
#include "core/io/file_access_pack.h" // PACK_HEADER_MAGIC, PACK_FORMAT_VERSION #include "core/io/file_access_pack.h" // PACK_HEADER_MAGIC, PACK_FORMAT_VERSION
@@ -66,12 +67,12 @@ class EditorExportSaveProxy {
public: public:
bool has_saved(const String &p_path) const { return saved_paths.has(p_path); } bool has_saved(const String &p_path) const { return saved_paths.has(p_path); }
Error save_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error save_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
if (tracking_saves) { if (tracking_saves) {
saved_paths.insert(p_path.simplify_path().trim_prefix("res://")); saved_paths.insert(p_path.simplify_path().trim_prefix("res://"));
} }
return save_func(p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed); return save_func(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta);
} }
EditorExportSaveProxy(EditorExportPlatform::EditorExportSaveFunction p_save_func, bool p_track_saves) : EditorExportSaveProxy(EditorExportPlatform::EditorExportSaveFunction p_save_func, bool p_track_saves) :
@@ -218,26 +219,6 @@ bool EditorExportPlatform::fill_log_messages(RichTextLabel *p_log, Error p_err)
return has_messages; return has_messages;
} }
bool EditorExportPlatform::_check_hash(const uint8_t *p_hash, const Vector<uint8_t> &p_data) {
if (p_hash == nullptr) {
return false;
}
unsigned char hash[16];
Error err = CryptoCore::md5(p_data.ptr(), p_data.size(), hash);
if (err != OK) {
return false;
}
for (int i = 0; i < 16; i++) {
if (p_hash[i] != hash[i]) {
return false;
}
}
return true;
}
Error EditorExportPlatform::_load_patches(const Vector<String> &p_patches) { Error EditorExportPlatform::_load_patches(const Vector<String> &p_patches) {
Error err = OK; Error err = OK;
if (!p_patches.is_empty()) { if (!p_patches.is_empty()) {
@@ -310,7 +291,7 @@ Error EditorExportPlatform::_encrypt_and_store_data(Ref<FileAccess> p_fd, const
return OK; return OK;
} }
Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatform::_save_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export."); ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export.");
PackData *pd = (PackData *)p_userdata; PackData *pd = (PackData *)p_userdata;
@@ -328,6 +309,7 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
sd.path_utf8 = simplified_path.trim_prefix("res://").utf8(); sd.path_utf8 = simplified_path.trim_prefix("res://").utf8();
sd.ofs = (pd->use_sparse_pck) ? 0 : pd->f->get_position(); sd.ofs = (pd->use_sparse_pck) ? 0 : pd->f->get_position();
sd.size = p_data.size(); sd.size = p_data.size();
sd.delta = p_delta;
Error err = _encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted); Error err = _encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted);
if (err != OK) { if (err != OK) {
return err; return err;
@@ -363,15 +345,67 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
return OK; return OK;
} }
Error EditorExportPlatform::_save_pack_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatform::_save_pack_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
if (_check_hash(PackedData::get_singleton()->get_file_hash(p_path), p_data)) { Ref<FileAccess> old_file = PackedData::get_singleton()->try_open_path(p_path);
return OK; if (old_file.is_null()) {
return _save_pack_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, false);
} }
return _save_pack_file(p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed); Vector<uint8_t> old_data = old_file->get_buffer(old_file->get_length());
// We can't rely on the MD5 as stored in the PCKs, since delta patches could have made it stale.
if (p_data == old_data) {
return OK; // Do nothing if the file hasn't changed.
} }
Error EditorExportPlatform::_save_zip_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { if (!p_preset->is_patch_delta_encoding_enabled()) {
return _save_pack_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, false);
}
bool delta = false;
for (const String &filter : p_preset->get_patch_delta_include_filter().split(",", false)) {
String filter_stripped = filter.strip_edges();
if (p_path.matchn(filter_stripped) || p_path.trim_prefix("res://").matchn(filter_stripped)) {
delta = true;
break;
}
}
for (const String &filter : p_preset->get_patch_delta_exclude_filter().split(",", false)) {
String filter_stripped = filter.strip_edges();
if (p_path.matchn(filter_stripped) || p_path.trim_prefix("res://").matchn(filter_stripped)) {
delta = false;
break;
}
}
Vector<uint8_t> patch_data = p_data;
if (delta) {
Error err = DeltaEncoding::encode_delta(old_data, p_data, patch_data, p_preset->get_patch_delta_zstd_level());
if (err != OK) {
return err;
}
int64_t reduction_bytes = MAX(0, p_data.size() - patch_data.size());
double reduction_ratio = reduction_bytes / (double)p_data.size();
if (reduction_ratio >= p_preset->get_patch_delta_min_reduction()) {
print_verbose(vformat("Used delta encoding for patch of \"%s\", resulting in a patch of %d bytes, which reduced the size by %.1f%% (%d bytes) compared to the actual file.", p_path, patch_data.size(), reduction_ratio * 100, reduction_bytes));
} else {
print_verbose(vformat("Skipped delta encoding for patch of \"%s\", as it resulted in a patch of %d bytes, which only reduced the size by %.1f%% (%d bytes) compared to the actual file.", p_path, patch_data.size(), reduction_ratio * 100, reduction_bytes));
patch_data = p_data;
delta = false;
}
} else {
print_verbose(vformat("Skipped delta encoding for patch of \"%s\", due to include/exclude filters.", p_path));
}
return _save_pack_file(p_preset, p_userdata, p_path, patch_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, delta);
}
Error EditorExportPlatform::_save_zip_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export."); ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export.");
const String path = simplify_path(p_path).replace_first("res://", ""); const String path = simplify_path(p_path).replace_first("res://", "");
@@ -403,12 +437,18 @@ Error EditorExportPlatform::_save_zip_file(void *p_userdata, const String &p_pat
return OK; return OK;
} }
Error EditorExportPlatform::_save_zip_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatform::_save_zip_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
if (_check_hash(PackedData::get_singleton()->get_file_hash(p_path), p_data)) { Ref<FileAccess> old_file = PackedData::get_singleton()->try_open_path(p_path);
return OK; if (old_file.is_valid()) {
Vector<uint8_t> old_data = old_file->get_buffer(old_file->get_length());
// We can't rely on the MD5 as stored in the PCKs, since delta patches could have made it stale.
if (p_data == old_data) {
return OK; // Do nothing if the file hasn't changed.
}
} }
return _save_zip_file(p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed); return _save_zip_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta);
} }
Ref<Texture2D> EditorExportPlatform::get_option_icon(int p_index) const { Ref<Texture2D> EditorExportPlatform::get_option_icon(int p_index) const {
@@ -1036,7 +1076,7 @@ Vector<String> EditorExportPlatform::get_forced_export_files(const Ref<EditorExp
return files; return files;
} }
Error EditorExportPlatform::_script_save_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatform::_script_save_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
Callable cb = ((ScriptCallbackData *)p_userdata)->file_cb; Callable cb = ((ScriptCallbackData *)p_userdata)->file_cb;
ERR_FAIL_COND_V(!cb.is_valid(), FAILED); ERR_FAIL_COND_V(!cb.is_valid(), FAILED);
@@ -1060,7 +1100,7 @@ Error EditorExportPlatform::_script_save_file(void *p_userdata, const String &p_
return (Error)ret.operator int(); return (Error)ret.operator int();
} }
Error EditorExportPlatform::_script_add_shared_object(void *p_userdata, const SharedObject &p_so) { Error EditorExportPlatform::_script_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
Callable cb = ((ScriptCallbackData *)p_userdata)->so_cb; Callable cb = ((ScriptCallbackData *)p_userdata)->so_cb;
if (!cb.is_valid()) { if (!cb.is_valid()) {
return OK; // Optional. return OK; // Optional.
@@ -1216,14 +1256,14 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
for (int i = 0; i < export_plugins.size(); i++) { for (int i = 0; i < export_plugins.size(); i++) {
if (p_so_func) { if (p_so_func) {
for (int j = 0; j < export_plugins[i]->shared_objects.size(); j++) { for (int j = 0; j < export_plugins[i]->shared_objects.size(); j++) {
err = p_so_func(p_udata, export_plugins[i]->shared_objects[j]); err = p_so_func(p_preset, p_udata, export_plugins[i]->shared_objects[j]);
if (err != OK) { if (err != OK) {
return err; return err;
} }
} }
} }
for (int j = 0; j < export_plugins[i]->extra_files.size(); j++) { for (int j = 0; j < export_plugins[i]->extra_files.size(); j++) {
err = save_proxy.save_file(p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, 0, paths.size(), enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, 0, paths.size(), enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1315,9 +1355,8 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
// for continue statements without accidentally skipping an increment. // for continue statements without accidentally skipping an increment.
int idx = total > 0 ? -1 : 0; int idx = total > 0 ? -1 : 0;
for (const String &E : paths) { for (const String &path : paths) {
idx++; idx++;
String path = E;
String type = ResourceLoader::get_resource_type(path); String type = ResourceLoader::get_resource_type(path);
bool has_import_file = FileAccess::exists(path + ".import"); bool has_import_file = FileAccess::exists(path + ".import");
@@ -1347,7 +1386,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
} }
if (p_so_func) { if (p_so_func) {
for (int j = 0; j < export_plugins[i]->shared_objects.size(); j++) { for (int j = 0; j < export_plugins[i]->shared_objects.size(); j++) {
err = p_so_func(p_udata, export_plugins[i]->shared_objects[j]); err = p_so_func(p_preset, p_udata, export_plugins[i]->shared_objects[j]);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1355,7 +1394,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
} }
for (int j = 0; j < export_plugins[i]->extra_files.size(); j++) { for (int j = 0; j < export_plugins[i]->extra_files.size(); j++) {
err = save_proxy.save_file(p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1385,7 +1424,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
if (importer_type == "keep") { if (importer_type == "keep") {
// Just keep file as-is. // Just keep file as-is.
Vector<uint8_t> array = FileAccess::get_file_as_bytes(path); Vector<uint8_t> array = FileAccess::get_file_as_bytes(path);
err = save_proxy.save_file(p_udata, path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
@@ -1427,13 +1466,13 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
sarr.resize(cs.size()); sarr.resize(cs.size());
memcpy(sarr.ptrw(), cs.ptr(), sarr.size()); memcpy(sarr.ptrw(), cs.ptr(), sarr.size());
err = save_proxy.save_file(p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
// Now actual remapped file: // Now actual remapped file:
sarr = FileAccess::get_file_as_bytes(export_path); sarr = FileAccess::get_file_as_bytes(export_path);
err = save_proxy.save_file(p_udata, export_path, sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, export_path, sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1461,14 +1500,14 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
if (remap == "path") { if (remap == "path") {
String remapped_path = config->get_value("remap", remap); String remapped_path = config->get_value("remap", remap);
Vector<uint8_t> array = FileAccess::get_file_as_bytes(remapped_path); Vector<uint8_t> array = FileAccess::get_file_as_bytes(remapped_path);
err = save_proxy.save_file(p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
} else if (remap.begins_with("path.")) { } else if (remap.begins_with("path.")) {
String feature = remap.get_slicec('.', 1); String feature = remap.get_slicec('.', 1);
if (remap_features.has(feature)) { if (remap_features.has(feature)) {
String remapped_path = config->get_value("remap", remap); String remapped_path = config->get_value("remap", remap);
Vector<uint8_t> array = FileAccess::get_file_as_bytes(remapped_path); Vector<uint8_t> array = FileAccess::get_file_as_bytes(remapped_path);
err = save_proxy.save_file(p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
} else { } else {
// Remove paths if feature not enabled. // Remove paths if feature not enabled.
config->erase_section_key("remap", remap); config->erase_section_key("remap", remap);
@@ -1494,7 +1533,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
sarr.resize(cs.size()); sarr.resize(cs.size());
memcpy(sarr.ptrw(), cs.ptr(), sarr.size()); memcpy(sarr.ptrw(), cs.ptr(), sarr.size());
err = save_proxy.save_file(p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
@@ -1515,7 +1554,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
} }
Vector<uint8_t> array = FileAccess::get_file_as_bytes(export_path); Vector<uint8_t> array = FileAccess::get_file_as_bytes(export_path);
err = save_proxy.save_file(p_udata, export_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, export_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1584,7 +1623,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
new_file.write[j] = utf8[j]; new_file.write[j] = utf8[j];
} }
err = save_proxy.save_file(p_udata, from + ".remap", new_file, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, from + ".remap", new_file, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1602,7 +1641,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
} else { } else {
array = FileAccess::get_file_as_bytes(forced_export[i]); array = FileAccess::get_file_as_bytes(forced_export[i]);
} }
err = save_proxy.save_file(p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1611,7 +1650,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
Dictionary int_export = get_internal_export_files(p_preset, p_debug); Dictionary int_export = get_internal_export_files(p_preset, p_debug);
for (const KeyValue<Variant, Variant> &int_export_kv : int_export) { for (const KeyValue<Variant, Variant> &int_export_kv : int_export) {
const PackedByteArray &array = int_export_kv.value; const PackedByteArray &array = int_export_kv.value;
err = save_proxy.save_file(p_udata, int_export_kv.key, array, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, int_export_kv.key, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1624,7 +1663,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
Vector<uint8_t> data = FileAccess::get_file_as_bytes(engine_cfb); Vector<uint8_t> data = FileAccess::get_file_as_bytes(engine_cfb);
DirAccess::remove_file_or_error(engine_cfb); DirAccess::remove_file_or_error(engine_cfb);
err = save_proxy.save_file(p_udata, "res://" + config_file, data, idx, total, enc_in_filters, enc_ex_filters, key, seed); err = save_proxy.save_file(p_preset, p_udata, "res://" + config_file, data, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1633,7 +1672,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
HashSet<String> currently_loaded_paths = PackedData::get_singleton()->get_file_paths(); HashSet<String> currently_loaded_paths = PackedData::get_singleton()->get_file_paths();
for (const String &path : currently_loaded_paths) { for (const String &path : currently_loaded_paths) {
if (!save_proxy.has_saved(path)) { if (!save_proxy.has_saved(path)) {
err = p_remove_func(p_udata, path); err = p_remove_func(p_preset, p_udata, path);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -1660,7 +1699,7 @@ Vector<uint8_t> EditorExportPlatform::_filter_extension_list_config_file(const S
return data; return data;
} }
Error EditorExportPlatform::_pack_add_shared_object(void *p_userdata, const SharedObject &p_so) { Error EditorExportPlatform::_pack_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
PackData *pack_data = (PackData *)p_userdata; PackData *pack_data = (PackData *)p_userdata;
if (pack_data->so_files) { if (pack_data->so_files) {
pack_data->so_files->push_back(p_so); pack_data->so_files->push_back(p_so);
@@ -1669,7 +1708,7 @@ Error EditorExportPlatform::_pack_add_shared_object(void *p_userdata, const Shar
return OK; return OK;
} }
Error EditorExportPlatform::_remove_pack_file(void *p_userdata, const String &p_path) { Error EditorExportPlatform::_remove_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path) {
PackData *pd = (PackData *)p_userdata; PackData *pd = (PackData *)p_userdata;
SavedData sd; SavedData sd;
@@ -1691,7 +1730,7 @@ Error EditorExportPlatform::_remove_pack_file(void *p_userdata, const String &p_
return OK; return OK;
} }
Error EditorExportPlatform::_zip_add_shared_object(void *p_userdata, const SharedObject &p_so) { Error EditorExportPlatform::_zip_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
ZipData *zip_data = (ZipData *)p_userdata; ZipData *zip_data = (ZipData *)p_userdata;
if (zip_data->so_files) { if (zip_data->so_files) {
zip_data->so_files->push_back(p_so); zip_data->so_files->push_back(p_so);
@@ -1996,6 +2035,9 @@ bool EditorExportPlatform::_encrypt_and_store_directory(Ref<FileAccess> p_fd, Pa
if (p_pack_data.file_ofs[i].removal) { if (p_pack_data.file_ofs[i].removal) {
flags |= PACK_FILE_REMOVAL; flags |= PACK_FILE_REMOVAL;
} }
if (p_pack_data.file_ofs[i].delta) {
flags |= PACK_FILE_DELTA;
}
fhead->store_32(flags); fhead->store_32(flags);
} }

27
editor/export/editor_export_platform.h
View File

@@ -53,9 +53,9 @@ protected:
static void _bind_methods(); static void _bind_methods();
public: public:
typedef Error (*EditorExportSaveFunction)(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); typedef Error (*EditorExportSaveFunction)(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
typedef Error (*EditorExportRemoveFunction)(void *p_userdata, const String &p_path); typedef Error (*EditorExportRemoveFunction)(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path);
typedef Error (*EditorExportSaveSharedObject)(void *p_userdata, const SharedObject &p_so); typedef Error (*EditorExportSaveSharedObject)(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
enum DebugFlags { enum DebugFlags {
DEBUG_FLAG_DUMB_CLIENT = 1, DEBUG_FLAG_DUMB_CLIENT = 1,
@@ -83,6 +83,7 @@ public:
uint64_t size = 0; uint64_t size = 0;
bool encrypted = false; bool encrypted = false;
bool removal = false; bool removal = false;
bool delta = false;
Vector<uint8_t> md5; Vector<uint8_t> md5;
CharString path_utf8; CharString path_utf8;
@@ -119,25 +120,23 @@ private:
void _export_find_customized_resources(const Ref<EditorExportPreset> &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet<String> &p_paths); void _export_find_customized_resources(const Ref<EditorExportPreset> &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet<String> &p_paths);
void _export_find_dependencies(const String &p_path, HashSet<String> &p_paths); void _export_find_dependencies(const String &p_path, HashSet<String> &p_paths);
static bool _check_hash(const uint8_t *p_hash, const Vector<uint8_t> &p_data); static Error _save_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _save_pack_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _pack_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
static Error _save_pack_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); static Error _remove_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path);
static Error _save_pack_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _pack_add_shared_object(void *p_userdata, const SharedObject &p_so);
static Error _remove_pack_file(void *p_userdata, const String &p_path); static Error _save_zip_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _save_zip_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _save_zip_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); static Error _zip_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
static Error _save_zip_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _zip_add_shared_object(void *p_userdata, const SharedObject &p_so);
struct ScriptCallbackData { struct ScriptCallbackData {
Callable file_cb; Callable file_cb;
Callable so_cb; Callable so_cb;
}; };
static Error _script_save_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); static Error _script_save_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _script_add_shared_object(void *p_userdata, const SharedObject &p_so); static Error _script_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
void _edit_files_with_filter(Ref<DirAccess> &da, const Vector<String> &p_filters, HashSet<String> &r_list, bool exclude); void _edit_files_with_filter(Ref<DirAccess> &da, const Vector<String> &p_filters, HashSet<String> &r_list, bool exclude);
void _edit_filter_list(HashSet<String> &r_list, const String &p_filter, bool exclude); void _edit_filter_list(HashSet<String> &r_list, const String &p_filter, bool exclude);

48
editor/export/editor_export_preset.cpp
View File

@@ -28,10 +28,11 @@
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/ /**************************************************************************/
#include "editor_export.h" #include "editor_export_preset.h"
#include "core/config/project_settings.h" #include "core/config/project_settings.h"
#include "core/io/dir_access.h" #include "core/io/dir_access.h"
#include "editor/export/editor_export.h"
#include "editor/settings/editor_settings.h" #include "editor/settings/editor_settings.h"
bool EditorExportPreset::_set(const StringName &p_name, const Variant &p_value) { bool EditorExportPreset::_set(const StringName &p_name, const Variant &p_value) {
@@ -440,6 +441,51 @@ Vector<String> EditorExportPreset::get_patches() const {
return patches; return patches;
} }
void EditorExportPreset::set_patch_delta_encoding_enabled(bool p_enable) {
patch_delta_encoding_enabled = p_enable;
EditorExport::singleton->save_presets();
}
bool EditorExportPreset::is_patch_delta_encoding_enabled() const {
return patch_delta_encoding_enabled;
}
void EditorExportPreset::set_patch_delta_zstd_level(int p_level) {
patch_delta_zstd_level = p_level;
EditorExport::singleton->save_presets();
}
int EditorExportPreset::get_patch_delta_zstd_level() const {
return patch_delta_zstd_level;
}
void EditorExportPreset::set_patch_delta_min_reduction(double p_ratio) {
patch_delta_min_reduction = p_ratio;
EditorExport::singleton->save_presets();
}
double EditorExportPreset::get_patch_delta_min_reduction() const {
return patch_delta_min_reduction;
}
void EditorExportPreset::set_patch_delta_include_filter(const String &p_filter) {
patch_delta_include_filter = p_filter;
EditorExport::singleton->save_presets();
}
String EditorExportPreset::get_patch_delta_include_filter() const {
return patch_delta_include_filter;
}
void EditorExportPreset::set_patch_delta_exclude_filter(const String &p_filter) {
patch_delta_exclude_filter = p_filter;
EditorExport::singleton->save_presets();
}
String EditorExportPreset::get_patch_delta_exclude_filter() const {
return patch_delta_exclude_filter;
}
void EditorExportPreset::set_custom_features(const String &p_custom_features) { void EditorExportPreset::set_custom_features(const String &p_custom_features) {
custom_features = p_custom_features; custom_features = p_custom_features;
EditorExport::singleton->save_presets(); EditorExport::singleton->save_presets();

22
editor/export/editor_export_preset.h
View File

@@ -73,6 +73,11 @@ private:
bool dedicated_server = false; bool dedicated_server = false;
Vector<String> patches; Vector<String> patches;
bool patch_delta_encoding_enabled = false;
int patch_delta_zstd_level = 19;
double patch_delta_min_reduction = 0.1;
String patch_delta_include_filter = "*";
String patch_delta_exclude_filter;
friend class EditorExport; friend class EditorExport;
friend class EditorExportPlatform; friend class EditorExportPlatform;
@@ -148,11 +153,28 @@ public:
void add_patch(const String &p_path, int p_at_pos = -1); void add_patch(const String &p_path, int p_at_pos = -1);
void set_patch(int p_index, const String &p_path); void set_patch(int p_index, const String &p_path);
String get_patch(int p_index); String get_patch(int p_index);
void remove_patch(int p_index); void remove_patch(int p_index);
void set_patches(const Vector<String> &p_patches); void set_patches(const Vector<String> &p_patches);
Vector<String> get_patches() const; Vector<String> get_patches() const;
void set_patch_delta_encoding_enabled(bool p_enable);
bool is_patch_delta_encoding_enabled() const;
void set_patch_delta_zstd_level(int p_level);
int get_patch_delta_zstd_level() const;
void set_patch_delta_min_reduction(double p_ratio);
double get_patch_delta_min_reduction() const;
void set_patch_delta_include_filter(const String &p_filter);
String get_patch_delta_include_filter() const;
void set_patch_delta_exclude_filter(const String &p_filter);
String get_patch_delta_exclude_filter() const;
void set_custom_features(const String &p_custom_features); void set_custom_features(const String &p_custom_features);
String get_custom_features() const; String get_custom_features() const;

135
editor/export/project_export.cpp
View File

@@ -51,11 +51,14 @@
#include "scene/gui/option_button.h" #include "scene/gui/option_button.h"
#include "scene/gui/popup_menu.h" #include "scene/gui/popup_menu.h"
#include "scene/gui/rich_text_label.h" #include "scene/gui/rich_text_label.h"
#include "scene/gui/spin_box.h"
#include "scene/gui/split_container.h" #include "scene/gui/split_container.h"
#include "scene/gui/tab_container.h" #include "scene/gui/tab_container.h"
#include "scene/gui/texture_rect.h" #include "scene/gui/texture_rect.h"
#include "scene/gui/tree.h" #include "scene/gui/tree.h"
#include <zstd.h>
void ProjectExportTextureFormatError::_on_fix_texture_format_pressed() { void ProjectExportTextureFormatError::_on_fix_texture_format_pressed() {
export_dialog->hide(); export_dialog->hide();
ProjectSettingsEditor *project_settings = EditorNode::get_singleton()->get_project_settings(); ProjectSettingsEditor *project_settings = EditorNode::get_singleton()->get_project_settings();
@@ -301,6 +304,19 @@ void ProjectExportDialog::_edit_preset(int p_index) {
exclude_filters->set_text(current->get_exclude_filter()); exclude_filters->set_text(current->get_exclude_filter());
server_strip_message->set_visible(current->get_export_filter() == EditorExportPreset::EXPORT_CUSTOMIZED); server_strip_message->set_visible(current->get_export_filter() == EditorExportPreset::EXPORT_CUSTOMIZED);
bool patch_delta_encoding_enabled = current->is_patch_delta_encoding_enabled();
patch_delta_encoding->set_pressed(patch_delta_encoding_enabled);
patch_delta_zstd_level->set_editable(patch_delta_encoding_enabled);
patch_delta_zstd_level->set_value(current->get_patch_delta_zstd_level());
patch_delta_min_reduction->set_editable(patch_delta_encoding_enabled);
patch_delta_min_reduction->set_value(current->get_patch_delta_min_reduction() * 100);
patch_delta_include_filter->set_editable(patch_delta_encoding_enabled);
patch_delta_exclude_filter->set_editable(patch_delta_encoding_enabled);
if (!updating_patch_delta_filters) {
patch_delta_include_filter->set_text(current->get_patch_delta_include_filter());
patch_delta_exclude_filter->set_text(current->get_patch_delta_exclude_filter());
}
patches->clear(); patches->clear();
TreeItem *patch_root = patches->create_item(); TreeItem *patch_root = patches->create_item();
Vector<String> patch_list = current->get_patches(); Vector<String> patch_list = current->get_patches();
@@ -745,6 +761,11 @@ void ProjectExportDialog::_duplicate_preset() {
preset->set_include_filter(current->get_include_filter()); preset->set_include_filter(current->get_include_filter());
preset->set_exclude_filter(current->get_exclude_filter()); preset->set_exclude_filter(current->get_exclude_filter());
preset->set_patches(current->get_patches()); preset->set_patches(current->get_patches());
preset->set_patch_delta_encoding_enabled(current->is_patch_delta_encoding_enabled());
preset->set_patch_delta_zstd_level(current->get_patch_delta_zstd_level());
preset->set_patch_delta_min_reduction(current->get_patch_delta_min_reduction());
preset->set_patch_delta_include_filter(current->get_patch_delta_include_filter());
preset->set_patch_delta_exclude_filter(current->get_patch_delta_exclude_filter());
preset->set_custom_features(current->get_custom_features()); preset->set_custom_features(current->get_custom_features());
preset->set_enc_in_filter(current->get_enc_in_filter()); preset->set_enc_in_filter(current->get_enc_in_filter());
preset->set_enc_ex_filter(current->get_enc_ex_filter()); preset->set_enc_ex_filter(current->get_enc_ex_filter());
@@ -1201,6 +1222,75 @@ void ProjectExportDialog::_set_file_export_mode(int p_id) {
_propagate_file_export_mode(include_files->get_root(), EditorExportPreset::MODE_FILE_NOT_CUSTOMIZED); _propagate_file_export_mode(include_files->get_root(), EditorExportPreset::MODE_FILE_NOT_CUSTOMIZED);
} }
void ProjectExportDialog::_patch_delta_encoding_changed(bool p_pressed) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_encoding_enabled(p_pressed);
_update_current_preset();
}
void ProjectExportDialog::_patch_delta_include_filter_changed(const String &p_filter) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_include_filter(patch_delta_include_filter->get_text());
updating_patch_delta_filters = true;
_update_current_preset();
updating_patch_delta_filters = false;
}
void ProjectExportDialog::_patch_delta_exclude_filter_changed(const String &p_filter) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_exclude_filter(patch_delta_exclude_filter->get_text());
updating_patch_delta_filters = true;
_update_current_preset();
updating_patch_delta_filters = false;
}
void ProjectExportDialog::_patch_delta_zstd_level_changed(double p_value) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_zstd_level((int)p_value);
_update_current_preset();
}
void ProjectExportDialog::_patch_delta_min_reduction_changed(double p_value) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_min_reduction(p_value / 100.0);
_update_current_preset();
}
void ProjectExportDialog::_patch_tree_button_clicked(Object *p_item, int p_column, int p_id, int p_mouse_button_index) { void ProjectExportDialog::_patch_tree_button_clicked(Object *p_item, int p_column, int p_id, int p_mouse_button_index) {
TreeItem *ti = Object::cast_to<TreeItem>(p_item); TreeItem *ti = Object::cast_to<TreeItem>(p_item);
@@ -1657,11 +1747,52 @@ ProjectExportDialog::ProjectExportDialog() {
exclude_filters); exclude_filters);
exclude_filters->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_filter_changed)); exclude_filters->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_filter_changed));
// Patch packages. // Patching.
VBoxContainer *patch_vb = memnew(VBoxContainer); VBoxContainer *patch_vb = memnew(VBoxContainer);
sections->add_child(patch_vb); sections->add_child(patch_vb);
patch_vb->set_name(TTR("Patches")); patch_vb->set_name(TTRC("Patching"));
patch_delta_encoding = memnew(CheckButton);
patch_delta_encoding->connect(SceneStringName(toggled), callable_mp(this, &ProjectExportDialog::_patch_delta_encoding_changed));
patch_delta_encoding->set_text(TTRC("Enable Delta Encoding"));
patch_delta_encoding->set_tooltip_text(TTRC("If checked, any change to a file already present in the base packs will be exported as the difference between the old file and the new file.\n"
"Enabling this comes at the cost of longer export times as well as longer load times for patched resources."));
patch_vb->add_child(patch_delta_encoding);
patch_delta_zstd_level = memnew(SpinBox);
patch_delta_zstd_level->set_min(ZSTD_minCLevel());
patch_delta_zstd_level->set_max(ZSTD_maxCLevel());
patch_delta_zstd_level->set_step(1);
patch_delta_zstd_level->set_tooltip_text(
vformat(TTR("The Zstandard compression level to use when generating delta-encoded patches.\n"
"Higher positive levels will reduce patch sizes, at the cost of longer export time, but do not affect the time it takes to apply patches.\n"
"Negative levels will reduce the time it takes to apply patches, at the cost of worse compression.\n"
"Levels above 19 require more memory both during export and when applying patches, usually for very little benefit.\n"
"Level 0 will cause Zstandard to use its default compression level, which is currently level %d."),
ZSTD_CLEVEL_DEFAULT));
patch_delta_zstd_level->connect(SceneStringName(value_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_zstd_level_changed));
patch_vb->add_margin_child(TTRC("Delta Encoding Compression Level"), patch_delta_zstd_level);
patch_delta_min_reduction = memnew(SpinBox);
patch_delta_min_reduction->set_min(0.0);
patch_delta_min_reduction->set_max(100.0);
patch_delta_min_reduction->set_step(1.0);
patch_delta_min_reduction->set_suffix("%");
patch_delta_min_reduction->set_tooltip_text(TTRC("How much smaller, when compared to the new file, a delta-encoded patch needs to be for it to be exported.\n"
"If the patch is not at least this much smaller, the new file will be exported as-is."));
patch_delta_min_reduction->connect(SceneStringName(value_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_min_reduction_changed));
patch_vb->add_margin_child(TTRC("Delta Encoding Minimum Size Reduction"), patch_delta_min_reduction);
patch_delta_include_filter = memnew(LineEdit);
patch_delta_include_filter->set_accessibility_name(TTRC("Delta Encoding Include Filters"));
patch_delta_include_filter->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_include_filter_changed));
patch_vb->add_margin_child(TTRC("Filters to include files/folders from being delta-encoded\n(comma-separated, e.g: *.gdc, scripts/*)"), patch_delta_include_filter);
patch_delta_exclude_filter = memnew(LineEdit);
patch_delta_exclude_filter->set_accessibility_name(TTRC("Delta Encoding Exclude Filters"));
patch_delta_exclude_filter->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_exclude_filter_changed));
patch_vb->add_margin_child(TTRC("Filters to exclude files/folders from being delta-encoded\n(comma-separated, e.g: *.ctex, textures/*)"), patch_delta_exclude_filter);
patches = memnew(Tree); patches = memnew(Tree);
patches->set_v_size_flags(Control::SIZE_EXPAND_FILL); patches->set_v_size_flags(Control::SIZE_EXPAND_FILL);

12
editor/export/project_export.h
View File

@@ -46,6 +46,7 @@ class OptionButton;
class PopupMenu; class PopupMenu;
class ProjectExportDialog; class ProjectExportDialog;
class RichTextLabel; class RichTextLabel;
class SpinBox;
class TabContainer; class TabContainer;
class Tree; class Tree;
class TreeItem; class TreeItem;
@@ -107,6 +108,11 @@ class ProjectExportDialog : public ConfirmationDialog {
RBSet<String> feature_set; RBSet<String> feature_set;
CheckButton *patch_delta_encoding = nullptr;
SpinBox *patch_delta_zstd_level = nullptr;
SpinBox *patch_delta_min_reduction = nullptr;
LineEdit *patch_delta_include_filter = nullptr;
LineEdit *patch_delta_exclude_filter = nullptr;
Tree *patches = nullptr; Tree *patches = nullptr;
int patch_index = -1; int patch_index = -1;
EditorFileDialog *patch_dialog = nullptr; EditorFileDialog *patch_dialog = nullptr;
@@ -158,6 +164,12 @@ class ProjectExportDialog : public ConfirmationDialog {
void _tree_popup_edited(bool p_arrow_clicked); void _tree_popup_edited(bool p_arrow_clicked);
void _set_file_export_mode(int p_id); void _set_file_export_mode(int p_id);
bool updating_patch_delta_filters = false;
void _patch_delta_encoding_changed(bool p_pressed);
void _patch_delta_include_filter_changed(const String &p_filter);
void _patch_delta_exclude_filter_changed(const String &p_filter);
void _patch_delta_zstd_level_changed(double p_value);
void _patch_delta_min_reduction_changed(double p_value);
void _patch_tree_button_clicked(Object *p_item, int p_column, int p_id, int p_mouse_button_index); void _patch_tree_button_clicked(Object *p_item, int p_column, int p_id, int p_mouse_button_index);
void _patch_tree_item_edited(); void _patch_tree_item_edited();
void _patch_file_selected(const String &p_path); void _patch_file_selected(const String &p_path);

10
platform/android/export/export_plugin.cpp
View File

@@ -767,7 +767,7 @@ Error EditorExportPlatformAndroid::store_in_apk(APKExportData *ed, const String
return OK; return OK;
} }
Error EditorExportPlatformAndroid::save_apk_so(void *p_userdata, const SharedObject &p_so) { Error EditorExportPlatformAndroid::save_apk_so(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
if (!p_so.path.get_file().begins_with("lib")) { if (!p_so.path.get_file().begins_with("lib")) {
String err = "Android .so file names must start with \"lib\", but got: " + p_so.path; String err = "Android .so file names must start with \"lib\", but got: " + p_so.path;
ERR_PRINT(err); ERR_PRINT(err);
@@ -801,14 +801,14 @@ Error EditorExportPlatformAndroid::save_apk_so(void *p_userdata, const SharedObj
return OK; return OK;
} }
Error EditorExportPlatformAndroid::save_apk_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatformAndroid::save_apk_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
APKExportData *ed = static_cast<APKExportData *>(p_userdata); APKExportData *ed = static_cast<APKExportData *>(p_userdata);
const String simplified_path = simplify_path(p_path); const String simplified_path = simplify_path(p_path);
Vector<uint8_t> enc_data; Vector<uint8_t> enc_data;
EditorExportPlatform::SavedData sd; EditorExportPlatform::SavedData sd;
Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, enc_data, sd); Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, enc_data, sd);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -822,11 +822,11 @@ Error EditorExportPlatformAndroid::save_apk_file(void *p_userdata, const String
return OK; return OK;
} }
Error EditorExportPlatformAndroid::ignore_apk_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error EditorExportPlatformAndroid::ignore_apk_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
return OK; return OK;
} }
Error EditorExportPlatformAndroid::copy_gradle_so(void *p_userdata, const SharedObject &p_so) { Error EditorExportPlatformAndroid::copy_gradle_so(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
ERR_FAIL_COND_V_MSG(!p_so.path.get_file().begins_with("lib"), FAILED, ERR_FAIL_COND_V_MSG(!p_so.path.get_file().begins_with("lib"), FAILED,
"Android .so file names must start with \"lib\", but got: " + p_so.path); "Android .so file names must start with \"lib\", but got: " + p_so.path);
Vector<ABI> abis = get_abis(); Vector<ABI> abis = get_abis();

8
platform/android/export/export_plugin.h
View File

@@ -151,13 +151,13 @@ class EditorExportPlatformAndroid : public EditorExportPlatform {
static Error store_in_apk(APKExportData *ed, const String &p_path, const Vector<uint8_t> &p_data, int compression_method = Z_DEFLATED); static Error store_in_apk(APKExportData *ed, const String &p_path, const Vector<uint8_t> &p_data, int compression_method = Z_DEFLATED);
static Error save_apk_so(void *p_userdata, const SharedObject &p_so); static Error save_apk_so(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
static Error save_apk_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); static Error save_apk_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error ignore_apk_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); static Error ignore_apk_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error copy_gradle_so(void *p_userdata, const SharedObject &p_so); static Error copy_gradle_so(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
bool _has_read_write_storage_permission(const Vector<String> &p_permissions); bool _has_read_write_storage_permission(const Vector<String> &p_permissions);

7
platform/android/export/gradle_export_util.cpp
View File

@@ -170,14 +170,14 @@ Error store_string_at_path(const String &p_path, const String &p_data) {
// It is used by the export_project_files method to save all the asset files into the gradle project. // It is used by the export_project_files method to save all the asset files into the gradle project.
// It's functionality mirrors that of the method save_apk_file. // It's functionality mirrors that of the method save_apk_file.
// This method will be called ONLY when gradle build is enabled. // This method will be called ONLY when gradle build is enabled.
Error rename_and_store_file_in_gradle_project(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) { Error rename_and_store_file_in_gradle_project(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
CustomExportData *export_data = static_cast<CustomExportData *>(p_userdata); CustomExportData *export_data = static_cast<CustomExportData *>(p_userdata);
const String simplified_path = EditorExportPlatform::simplify_path(p_path); const String simplified_path = EditorExportPlatform::simplify_path(p_path);
Vector<uint8_t> enc_data; Vector<uint8_t> enc_data;
EditorExportPlatform::SavedData sd; EditorExportPlatform::SavedData sd;
Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, enc_data, sd); Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, enc_data, sd);
if (err != OK) { if (err != OK) {
return err; return err;
} }
@@ -398,7 +398,7 @@ String _get_application_tag(const Ref<EditorExportPlatform> &p_export_platform,
return manifest_application_text; return manifest_application_text;
} }
Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, Vector<uint8_t> &r_enc_data, EditorExportPlatform::SavedData &r_sd) { Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta, Vector<uint8_t> &r_enc_data, EditorExportPlatform::SavedData &r_sd) {
Error err = OK; Error err = OK;
Ref<FileAccess> ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export", "tmp", false, &err); Ref<FileAccess> ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export", "tmp", false, &err);
if (err != OK) { if (err != OK) {
@@ -407,6 +407,7 @@ Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p
r_sd.path_utf8 = p_simplified_path.trim_prefix("res://").utf8(); r_sd.path_utf8 = p_simplified_path.trim_prefix("res://").utf8();
r_sd.ofs = 0; r_sd.ofs = 0;
r_sd.size = p_data.size(); r_sd.size = p_data.size();
r_sd.delta = p_delta;
err = EditorExportPlatform::_encrypt_and_store_data(ftmp, p_simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, r_sd.encrypted); err = EditorExportPlatform::_encrypt_and_store_data(ftmp, p_simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, r_sd.encrypted);
if (err != OK) { if (err != OK) {
return err; return err;

4
platform/android/export/gradle_export_util.h
View File

@@ -85,7 +85,7 @@ int _get_app_category_value(int category_index);
String _get_app_category_label(int category_index); String _get_app_category_label(int category_index);
Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, Vector<uint8_t> &r_enc_data, EditorExportPlatform::SavedData &r_sd); Error _store_temp_file(const String &p_simplified_path, const Vector<uint8_t> &p_data, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta, Vector<uint8_t> &r_enc_data, EditorExportPlatform::SavedData &r_sd);
// Utility method used to create a directory. // Utility method used to create a directory.
Error create_directory(const String &p_dir); Error create_directory(const String &p_dir);
@@ -103,7 +103,7 @@ Error store_string_at_path(const String &p_path, const String &p_data);
// It is used by the export_project_files method to save all the asset files into the gradle project. // It is used by the export_project_files method to save all the asset files into the gradle project.
// It's functionality mirrors that of the method save_apk_file. // It's functionality mirrors that of the method save_apk_file.
// This method will be called ONLY when gradle build is enabled. // This method will be called ONLY when gradle build is enabled.
Error rename_and_store_file_in_gradle_project(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed); Error rename_and_store_file_in_gradle_project(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
// Creates strings.xml files inside the gradle project for different locales. // Creates strings.xml files inside the gradle project for different locales.
Error _create_project_name_strings_files(const Ref<EditorExportPreset> &p_preset, const String &p_project_name, const String &p_gradle_build_dir, const Dictionary &p_appnames); Error _create_project_name_strings_files(const Ref<EditorExportPreset> &p_preset, const String &p_project_name, const String &p_gradle_build_dir, const Dictionary &p_appnames);