Merge pull request #69119 from lawnjelly/faster_variant_parser_master

Add readahead to VariantParser [4.x]

core/variant/variant_parser.cpp

@@ -35,37 +35,76 @@
 #include "core/os/keyboard.h"
 #include "core/string/string_buffer.h"
 
-char32_t VariantParser::StreamFile::get_char() {
+char32_t VariantParser::Stream::get_char() {
-	return f->get_8();
+	// is within buffer?
+	if (readahead_pointer < readahead_filled) {
+		return readahead_buffer[readahead_pointer++];
+	}
+
+	// attempt to readahead
+	readahead_filled = _read_buffer(readahead_buffer, READAHEAD_SIZE);
+	if (readahead_filled) {
+		readahead_pointer = 0;
+	} else {
+		// EOF
+		readahead_pointer = 1;
+		eof = true;
+		return 0;
+	}
+	return get_char();
 }
 
 bool VariantParser::StreamFile::is_utf8() const {
 	return true;
 }
 
-bool VariantParser::StreamFile::is_eof() const {
+uint32_t VariantParser::StreamFile::_read_buffer(char32_t *p_buffer, uint32_t p_num_chars) {
-	return f->eof_reached();
+	// The buffer is assumed to include at least one character (for null terminator)
+	ERR_FAIL_COND_V(!p_num_chars, 0);
+
+	uint8_t *temp = (uint8_t *)alloca(p_num_chars);
+	uint64_t num_read = f->get_buffer(temp, p_num_chars);
+	ERR_FAIL_COND_V(num_read == UINT64_MAX, 0);
+
+	// translate to wchar
+	for (uint32_t n = 0; n < num_read; n++) {
+		p_buffer[n] = temp[n];
 	}
 
-char32_t VariantParser::StreamString::get_char() {
+	// could be less than p_num_chars, or zero
-	if (pos > s.length()) {
+	return num_read;
-		return 0;
-	} else if (pos == s.length()) {
-		// You need to try to read again when you have reached the end for EOF to be reported,
-		// so this works the same as files (like StreamFile does)
-		pos++;
-		return 0;
-	} else {
-		return s[pos++];
-	}
 }
 
 bool VariantParser::StreamString::is_utf8() const {
 	return false;
 }
 
-bool VariantParser::StreamString::is_eof() const {
+uint32_t VariantParser::StreamString::_read_buffer(char32_t *p_buffer, uint32_t p_num_chars) {
-	return pos > s.length();
+	// The buffer is assumed to include at least one character (for null terminator)
+	ERR_FAIL_COND_V(!p_num_chars, 0);
+
+	int available = MAX(s.length() - pos, 0);
+	if (available >= (int)p_num_chars) {
+		const char32_t *src = s.ptr();
+		src += pos;
+		memcpy(p_buffer, src, p_num_chars * sizeof(char32_t));
+		pos += p_num_chars;
+
+		return p_num_chars;
+	}
+
+	// going to reach EOF
+	if (available) {
+		const char32_t *src = s.ptr();
+		src += pos;
+		memcpy(p_buffer, src, available * sizeof(char32_t));
+		pos += available;
+	}
+
+	// add a zero
+	p_buffer[available] = 0;
+
+	return available;
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////

core/variant/variant_parser.h

@@ -38,34 +38,50 @@
 class VariantParser {
 public:
 	struct Stream {
-		virtual char32_t get_char() = 0;
+	private:
-		virtual bool is_utf8() const = 0;
+		enum { READAHEAD_SIZE = 2048 };
-		virtual bool is_eof() const = 0;
+		char32_t readahead_buffer[READAHEAD_SIZE];
+		uint32_t readahead_pointer = 0;
+		uint32_t readahead_filled = 0;
+		bool eof = false;
 
+	protected:
+		virtual uint32_t _read_buffer(char32_t *p_buffer, uint32_t p_num_chars) = 0;
+
+	public:
 		char32_t saved = 0;
 
+		char32_t get_char();
+		virtual bool is_utf8() const = 0;
+		bool is_eof() const { return eof; }
+
 		Stream() {}
 		virtual ~Stream() {}
 	};
 
 	struct StreamFile : public Stream {
+	protected:
+		virtual uint32_t _read_buffer(char32_t *p_buffer, uint32_t p_num_chars) override;
+
+	public:
 		Ref<FileAccess> f;
 
-		virtual char32_t get_char() override;
 		virtual bool is_utf8() const override;
-		virtual bool is_eof() const override;
 
 		StreamFile() {}
 	};
 
 	struct StreamString : public Stream {
 		String s;
+
+	private:
 		int pos = 0;
 
-		virtual char32_t get_char() override;
+	protected:
-		virtual bool is_utf8() const override;
+		virtual uint32_t _read_buffer(char32_t *p_buffer, uint32_t p_num_chars) override;
-		virtual bool is_eof() const override;
 
+	public:
+		virtual bool is_utf8() const override;
 		StreamString() {}
 	};