You've already forked godot
mirror of
https://github.com/godotengine/godot.git
synced 2025-11-05 12:10:55 +00:00
Redo how the remote filesystem works
Instead of reading files over the network, the new version uses a local file cache and only updates files when it changes. The original remote filesystem was created 14 years ago, when ethernet was faster than hard drives or even flash. Also, mobile devices have a very small amount of storage. Nowadays, this is no longer the case so the approach is changed to using a persistent cache in the target device. Co-authored-by: m4gr3d
This commit is contained in:
Juan Linietsky
@@ -32,273 +32,230 @@
|
||||
|
||||
#include "../editor_settings.h"
|
||||
#include "core/io/marshalls.h"
|
||||
#include "editor/editor_node.h"
|
||||
#include "editor/export/editor_export_platform.h"
|
||||
|
||||
//#define DEBUG_PRINT(m_p) print_line(m_p)
|
||||
//#define DEBUG_TIME(m_what) printf("MS: %s - %lu\n", m_what, OS::get_singleton()->get_ticks_usec());
|
||||
#define FILESYSTEM_PROTOCOL_VERSION 1
|
||||
#define PASSWORD_LENGTH 32
|
||||
#define MAX_FILE_BUFFER_SIZE 100 * 1024 * 1024 // 100mb max file buffer size (description of files to update, compressed).
|
||||
|
||||
#define DEBUG_PRINT(m_what)
|
||||
#define DEBUG_TIME(m_what)
|
||||
|
||||
void EditorFileServer::_close_client(ClientData *cd) {
|
||||
cd->connection->disconnect_from_host();
|
||||
{
|
||||
MutexLock lock(cd->efs->wait_mutex);
|
||||
cd->efs->to_wait.insert(cd->thread);
|
||||
static void _add_file(String f, const uint64_t &p_modified_time, HashMap<String, uint64_t> &files_to_send, HashMap<String, uint64_t> &cached_files) {
|
||||
f = f.replace_first("res://", ""); // remove res://
|
||||
const uint64_t *cached_mt = cached_files.getptr(f);
|
||||
if (cached_mt && *cached_mt == p_modified_time) {
|
||||
// File is good, skip it.
|
||||
cached_files.erase(f); // Erase to mark this file as existing. Remaning files not added to files_to_send will be considered erased here, so they need to be erased in the client too.
|
||||
return;
|
||||
}
|
||||
while (cd->files.size()) {
|
||||
cd->files.remove(cd->files.begin());
|
||||
}
|
||||
memdelete(cd);
|
||||
files_to_send.insert(f, p_modified_time);
|
||||
}
|
||||
|
||||
void EditorFileServer::_subthread_start(void *s) {
|
||||
ClientData *cd = static_cast<ClientData *>(s);
|
||||
void EditorFileServer::_scan_files_changed(EditorFileSystemDirectory *efd, const Vector<String> &p_tags, HashMap<String, uint64_t> &files_to_send, HashMap<String, uint64_t> &cached_files) {
|
||||
for (int i = 0; i < efd->get_file_count(); i++) {
|
||||
String f = efd->get_file_path(i);
|
||||
if (FileAccess::exists(f + ".import")) {
|
||||
// is imported, determine what to do
|
||||
// Todo the modified times of remapped files should most likely be kept in EditorFileSystem to speed this up in the future.
|
||||
Ref<ConfigFile> cf;
|
||||
cf.instantiate();
|
||||
Error err = cf->load(f + ".import");
|
||||
|
||||
cd->connection->set_no_delay(true);
|
||||
uint8_t buf4[8];
|
||||
Error err = cd->connection->get_data(buf4, 4);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
ERR_CONTINUE(err != OK);
|
||||
{
|
||||
uint64_t mt = FileAccess::get_modified_time(f + ".import");
|
||||
_add_file(f + ".import", mt, files_to_send, cached_files);
|
||||
}
|
||||
|
||||
if (!cf->has_section("remap")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> remaps;
|
||||
cf->get_section_keys("remap", &remaps);
|
||||
|
||||
for (const String &remap : remaps) {
|
||||
if (remap == "path") {
|
||||
String remapped_path = cf->get_value("remap", remap);
|
||||
uint64_t mt = FileAccess::get_modified_time(remapped_path);
|
||||
_add_file(remapped_path, mt, files_to_send, cached_files);
|
||||
} else if (remap.begins_with("path.")) {
|
||||
String feature = remap.get_slice(".", 1);
|
||||
if (p_tags.find(feature) != -1) {
|
||||
String remapped_path = cf->get_value("remap", remap);
|
||||
uint64_t mt = FileAccess::get_modified_time(remapped_path);
|
||||
_add_file(remapped_path, mt, files_to_send, cached_files);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uint64_t mt = efd->get_file_modified_time(i);
|
||||
_add_file(f, mt, files_to_send, cached_files);
|
||||
}
|
||||
}
|
||||
|
||||
int passlen = decode_uint32(buf4);
|
||||
for (int i = 0; i < efd->get_subdir_count(); i++) {
|
||||
_scan_files_changed(efd->get_subdir(i), p_tags, files_to_send, cached_files);
|
||||
}
|
||||
}
|
||||
|
||||
if (passlen > 512) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(passlen > 512);
|
||||
} else if (passlen > 0) {
|
||||
Vector<char> passutf8;
|
||||
passutf8.resize(passlen + 1);
|
||||
err = cd->connection->get_data((uint8_t *)passutf8.ptr(), passlen);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
passutf8.write[passlen] = 0;
|
||||
String s2;
|
||||
s2.parse_utf8(passutf8.ptr());
|
||||
if (s2 != cd->efs->password) {
|
||||
encode_uint32(ERR_INVALID_DATA, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
OS::get_singleton()->delay_usec(1000000);
|
||||
_close_client(cd);
|
||||
ERR_PRINT("CLIENT PASSWORD MISMATCH");
|
||||
ERR_FAIL();
|
||||
static void _add_custom_file(const String f, HashMap<String, uint64_t> &files_to_send, HashMap<String, uint64_t> &cached_files) {
|
||||
if (!FileAccess::exists(f)) {
|
||||
return;
|
||||
}
|
||||
_add_file(f, FileAccess::get_modified_time(f), files_to_send, cached_files);
|
||||
}
|
||||
|
||||
void EditorFileServer::poll() {
|
||||
if (!active) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!server->is_connection_available()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Ref<StreamPeerTCP> tcp_peer = server->take_connection();
|
||||
ERR_FAIL_COND(tcp_peer.is_null());
|
||||
|
||||
// Got a connection!
|
||||
EditorProgress pr("updating_remote_file_system", TTR("Updating assets on target device:"), 105);
|
||||
|
||||
pr.step(TTR("Syncinc headers"), 0, true);
|
||||
print_verbose("EFS: Connecting taken!");
|
||||
char header[4];
|
||||
Error err = tcp_peer->get_data((uint8_t *)&header, 4);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
ERR_FAIL_COND(header[0] != 'G');
|
||||
ERR_FAIL_COND(header[1] != 'R');
|
||||
ERR_FAIL_COND(header[2] != 'F');
|
||||
ERR_FAIL_COND(header[3] != 'S');
|
||||
|
||||
uint32_t protocol_version = tcp_peer->get_u32();
|
||||
ERR_FAIL_COND(protocol_version != FILESYSTEM_PROTOCOL_VERSION);
|
||||
|
||||
char cpassword[PASSWORD_LENGTH + 1];
|
||||
err = tcp_peer->get_data((uint8_t *)cpassword, PASSWORD_LENGTH);
|
||||
cpassword[PASSWORD_LENGTH] = 0;
|
||||
ERR_FAIL_COND(err != OK);
|
||||
print_verbose("EFS: Got password: " + String(cpassword));
|
||||
ERR_FAIL_COND_MSG(password != cpassword, "Client disconnected because password mismatch.");
|
||||
|
||||
uint32_t tag_count = tcp_peer->get_u32();
|
||||
print_verbose("EFS: Getting tags: " + itos(tag_count));
|
||||
|
||||
ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
|
||||
Vector<String> tags;
|
||||
for (uint32_t i = 0; i < tag_count; i++) {
|
||||
String tag = tcp_peer->get_utf8_string();
|
||||
print_verbose("EFS: tag #" + itos(i) + ": " + tag);
|
||||
ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
|
||||
tags.push_back(tag);
|
||||
}
|
||||
|
||||
uint32_t file_buffer_decompressed_size = tcp_peer->get_32();
|
||||
HashMap<String, uint64_t> cached_files;
|
||||
|
||||
if (file_buffer_decompressed_size > 0) {
|
||||
pr.step(TTR("Getting remote file system"), 1, true);
|
||||
|
||||
// Got files cached by client.
|
||||
uint32_t file_buffer_size = tcp_peer->get_32();
|
||||
print_verbose("EFS: Getting file buffer: compressed - " + String::humanize_size(file_buffer_size) + " decompressed: " + String::humanize_size(file_buffer_decompressed_size));
|
||||
|
||||
ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
|
||||
ERR_FAIL_COND(file_buffer_size > MAX_FILE_BUFFER_SIZE);
|
||||
LocalVector<uint8_t> file_buffer;
|
||||
file_buffer.resize(file_buffer_size);
|
||||
LocalVector<uint8_t> file_buffer_decompressed;
|
||||
file_buffer_decompressed.resize(file_buffer_decompressed_size);
|
||||
|
||||
err = tcp_peer->get_data(file_buffer.ptr(), file_buffer_size);
|
||||
|
||||
pr.step(TTR("Decompressing remote file system"), 2, true);
|
||||
|
||||
ERR_FAIL_COND(err != OK);
|
||||
// Decompress the text with all the files
|
||||
Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD);
|
||||
String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size());
|
||||
Vector<String> files = files_text.split("\n");
|
||||
|
||||
print_verbose("EFS: Total cached files received: " + itos(files.size()));
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
if (files[i].get_slice_count("::") != 2) {
|
||||
continue;
|
||||
}
|
||||
String file = files[i].get_slice("::", 0);
|
||||
uint64_t modified_time = files[i].get_slice("::", 1).to_int();
|
||||
|
||||
cached_files.insert(file, modified_time);
|
||||
}
|
||||
} else {
|
||||
if (!cd->efs->password.is_empty()) {
|
||||
encode_uint32(ERR_INVALID_DATA, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
OS::get_singleton()->delay_usec(1000000);
|
||||
_close_client(cd);
|
||||
ERR_PRINT("CLIENT PASSWORD MISMATCH (should be empty!)");
|
||||
ERR_FAIL();
|
||||
// Client does not have any files stored.
|
||||
}
|
||||
|
||||
pr.step(TTR("Scanning for local changes"), 3, true);
|
||||
|
||||
print_verbose("EFS: Scanning changes:");
|
||||
|
||||
HashMap<String, uint64_t> files_to_send;
|
||||
// Scan files to send.
|
||||
_scan_files_changed(EditorFileSystem::get_singleton()->get_filesystem(), tags, files_to_send, cached_files);
|
||||
// Add forced export files
|
||||
Vector<String> forced_export = EditorExportPlatform::get_forced_export_files();
|
||||
for (int i = 0; i < forced_export.size(); i++) {
|
||||
_add_custom_file(forced_export[i], files_to_send, cached_files);
|
||||
}
|
||||
|
||||
_add_custom_file("res://project.godot", files_to_send, cached_files);
|
||||
// Check which files were removed and also add them
|
||||
for (KeyValue<String, uint64_t> K : cached_files) {
|
||||
if (!files_to_send.has(K.key)) {
|
||||
files_to_send.insert(K.key, 0); //0 means removed
|
||||
}
|
||||
}
|
||||
|
||||
encode_uint32(OK, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
tcp_peer->put_32(files_to_send.size());
|
||||
|
||||
while (!cd->quit) {
|
||||
//wait for ID
|
||||
err = cd->connection->get_data(buf4, 4);
|
||||
DEBUG_TIME("get_data")
|
||||
print_verbose("EFS: Sending list of changed files.");
|
||||
pr.step(TTR("Sending list of changed files:"), 4, true);
|
||||
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
int id = decode_uint32(buf4);
|
||||
|
||||
//wait for command
|
||||
err = cd->connection->get_data(buf4, 4);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
int cmd = decode_uint32(buf4);
|
||||
|
||||
switch (cmd) {
|
||||
case FileAccessNetwork::COMMAND_FILE_EXISTS:
|
||||
case FileAccessNetwork::COMMAND_GET_MODTIME:
|
||||
case FileAccessNetwork::COMMAND_OPEN_FILE: {
|
||||
DEBUG_TIME("open_file")
|
||||
err = cd->connection->get_data(buf4, 4);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
|
||||
int namelen = decode_uint32(buf4);
|
||||
Vector<char> fileutf8;
|
||||
fileutf8.resize(namelen + 1);
|
||||
err = cd->connection->get_data((uint8_t *)fileutf8.ptr(), namelen);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
fileutf8.write[namelen] = 0;
|
||||
String s2;
|
||||
s2.parse_utf8(fileutf8.ptr());
|
||||
|
||||
if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) {
|
||||
print_verbose("FILE EXISTS: " + s2);
|
||||
}
|
||||
if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) {
|
||||
print_verbose("MOD TIME: " + s2);
|
||||
}
|
||||
if (cmd == FileAccessNetwork::COMMAND_OPEN_FILE) {
|
||||
print_verbose("OPEN: " + s2);
|
||||
}
|
||||
|
||||
if (!s2.begins_with("res://")) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(!s2.begins_with("res://"));
|
||||
}
|
||||
ERR_CONTINUE(cd->files.has(id));
|
||||
|
||||
if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) {
|
||||
encode_uint32(id, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(FileAccessNetwork::RESPONSE_FILE_EXISTS, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(FileAccess::exists(s2), buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
DEBUG_TIME("open_file_end")
|
||||
break;
|
||||
}
|
||||
|
||||
if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) {
|
||||
encode_uint32(id, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(FileAccessNetwork::RESPONSE_GET_MODTIME, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint64(FileAccess::get_modified_time(s2), buf4);
|
||||
cd->connection->put_data(buf4, 8);
|
||||
DEBUG_TIME("open_file_end")
|
||||
break;
|
||||
}
|
||||
|
||||
Ref<FileAccess> fa = FileAccess::open(s2, FileAccess::READ);
|
||||
if (fa.is_null()) {
|
||||
//not found, continue
|
||||
encode_uint32(id, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(ERR_FILE_NOT_FOUND, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
DEBUG_TIME("open_file_end")
|
||||
break;
|
||||
}
|
||||
|
||||
encode_uint32(id, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(OK, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint64(fa->get_length(), buf4);
|
||||
cd->connection->put_data(buf4, 8);
|
||||
|
||||
cd->files[id] = fa;
|
||||
DEBUG_TIME("open_file_end")
|
||||
|
||||
} break;
|
||||
case FileAccessNetwork::COMMAND_READ_BLOCK: {
|
||||
err = cd->connection->get_data(buf4, 8);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
|
||||
ERR_CONTINUE(!cd->files.has(id));
|
||||
|
||||
uint64_t offset = decode_uint64(buf4);
|
||||
|
||||
err = cd->connection->get_data(buf4, 4);
|
||||
if (err != OK) {
|
||||
_close_client(cd);
|
||||
ERR_FAIL_COND(err != OK);
|
||||
}
|
||||
|
||||
int blocklen = decode_uint32(buf4);
|
||||
ERR_CONTINUE(blocklen > (16 * 1024 * 1024));
|
||||
|
||||
cd->files[id]->seek(offset);
|
||||
Vector<uint8_t> buf;
|
||||
buf.resize(blocklen);
|
||||
uint32_t read = cd->files[id]->get_buffer(buf.ptrw(), blocklen);
|
||||
|
||||
print_verbose("GET BLOCK - offset: " + itos(offset) + ", blocklen: " + itos(blocklen));
|
||||
|
||||
//not found, continue
|
||||
encode_uint32(id, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint32(FileAccessNetwork::RESPONSE_DATA, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
encode_uint64(offset, buf4);
|
||||
cd->connection->put_data(buf4, 8);
|
||||
encode_uint32(read, buf4);
|
||||
cd->connection->put_data(buf4, 4);
|
||||
cd->connection->put_data(buf.ptr(), read);
|
||||
|
||||
} break;
|
||||
case FileAccessNetwork::COMMAND_CLOSE: {
|
||||
print_verbose("CLOSED");
|
||||
ERR_CONTINUE(!cd->files.has(id));
|
||||
cd->files.erase(id);
|
||||
} break;
|
||||
}
|
||||
// Send list of changed files first, to ensure that if connecting breaks, the client is not found in a broken state.
|
||||
for (KeyValue<String, uint64_t> K : files_to_send) {
|
||||
tcp_peer->put_utf8_string(K.key);
|
||||
tcp_peer->put_64(K.value);
|
||||
}
|
||||
|
||||
_close_client(cd);
|
||||
}
|
||||
print_verbose("EFS: Sending " + itos(files_to_send.size()) + " files.");
|
||||
|
||||
void EditorFileServer::_thread_start(void *s) {
|
||||
EditorFileServer *self = static_cast<EditorFileServer *>(s);
|
||||
while (!self->quit) {
|
||||
if (self->cmd == CMD_ACTIVATE) {
|
||||
self->server->listen(self->port);
|
||||
self->active = true;
|
||||
self->cmd = CMD_NONE;
|
||||
} else if (self->cmd == CMD_STOP) {
|
||||
self->server->stop();
|
||||
self->active = false;
|
||||
self->cmd = CMD_NONE;
|
||||
int idx = 0;
|
||||
for (KeyValue<String, uint64_t> K : files_to_send) {
|
||||
pr.step(TTR("Sending file: ") + K.key.get_file(), 5 + idx * 100 / files_to_send.size(), false);
|
||||
idx++;
|
||||
|
||||
if (K.value == 0 || !FileAccess::exists("res://" + K.key)) { // File was removed
|
||||
continue;
|
||||
}
|
||||
|
||||
if (self->active) {
|
||||
if (self->server->is_connection_available()) {
|
||||
ClientData *cd = memnew(ClientData);
|
||||
cd->connection = self->server->take_connection();
|
||||
cd->efs = self;
|
||||
cd->quit = false;
|
||||
cd->thread = memnew(Thread);
|
||||
cd->thread->start(_subthread_start, cd);
|
||||
}
|
||||
}
|
||||
|
||||
self->wait_mutex.lock();
|
||||
while (self->to_wait.size()) {
|
||||
Thread *w = *self->to_wait.begin();
|
||||
self->to_wait.erase(w);
|
||||
self->wait_mutex.unlock();
|
||||
w->wait_to_finish();
|
||||
self->wait_mutex.lock();
|
||||
}
|
||||
self->wait_mutex.unlock();
|
||||
|
||||
OS::get_singleton()->delay_usec(100000);
|
||||
Vector<uint8_t> array = FileAccess::_get_file_as_bytes("res://" + K.key);
|
||||
tcp_peer->put_64(array.size());
|
||||
tcp_peer->put_data(array.ptr(), array.size());
|
||||
ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
|
||||
}
|
||||
|
||||
tcp_peer->put_data((const uint8_t *)"GEND", 4); // End marker.
|
||||
|
||||
print_verbose("EFS: Done.");
|
||||
}
|
||||
|
||||
void EditorFileServer::start() {
|
||||
stop();
|
||||
if (active) {
|
||||
stop();
|
||||
}
|
||||
port = EDITOR_GET("filesystem/file_server/port");
|
||||
password = EDITOR_GET("filesystem/file_server/password");
|
||||
cmd = CMD_ACTIVATE;
|
||||
Error err = server->listen(port);
|
||||
ERR_FAIL_COND_MSG(err != OK, "EditorFileServer: Unable to listen on port " + itos(port));
|
||||
active = true;
|
||||
}
|
||||
|
||||
bool EditorFileServer::is_active() const {
|
||||
@@ -306,18 +263,19 @@ bool EditorFileServer::is_active() const {
|
||||
}
|
||||
|
||||
void EditorFileServer::stop() {
|
||||
cmd = CMD_STOP;
|
||||
if (active) {
|
||||
server->stop();
|
||||
active = false;
|
||||
}
|
||||
}
|
||||
|
||||
EditorFileServer::EditorFileServer() {
|
||||
server.instantiate();
|
||||
thread.start(_thread_start, this);
|
||||
|
||||
EDITOR_DEF("filesystem/file_server/port", 6010);
|
||||
EDITOR_DEF("filesystem/file_server/password", "");
|
||||
}
|
||||
|
||||
EditorFileServer::~EditorFileServer() {
|
||||
quit = true;
|
||||
thread.wait_to_finish();
|
||||
stop();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user