WIP New GLES3 Shader Compiler

Uses versions and specializations (more similar to RenderingDevice version)

gles3_builders.py

@@ -6,16 +6,12 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki
 from platform_methods import subprocess_main
 
 
-class LegacyGLHeaderStruct:
+class GLES3HeaderStruct:
     def __init__(self):
         self.vertex_lines = []
         self.fragment_lines = []
         self.uniforms = []
-        self.attributes = []
-        self.feedbacks = []
         self.fbos = []
-        self.conditionals = []
-        self.enums = {}
         self.texunits = []
         self.texunit_names = []
         self.ubos = []
@@ -28,22 +24,65 @@ class LegacyGLHeaderStruct:
         self.line_offset = 0
         self.vertex_offset = 0
         self.fragment_offset = 0
+        self.variant_defines=[]
+        self.variant_names=[]
+        self.specialization_names=[]
+        self.specialization_values=[]
 
 
-def include_file_in_legacygl_header(filename, header_data, depth):
+def include_file_in_gles3_header(filename, header_data, depth):
     fs = open(filename, "r")
     line = fs.readline()
 
     while line:
 
-        if line.find("[vertex]") != -1:
+        if line.find("=") != -1 and header_data.reading=="": 
+            # Mode
+            eqpos = line.find("=")
+            defname = line[:eqpos].strip().upper()
+            define = line[eqpos+1:].strip()
+            header_data.variant_names.append( defname )
+            header_data.variant_defines.append( define )
+            line = fs.readline()
+            header_data.line_offset += 1
+            header_data.vertex_offset = header_data.line_offset
+            continue
+
+        if line.find("=") != -1 and header_data.reading=="specializations": 
+            # Specialization
+            eqpos = line.find("=")
+            specname = line[:eqpos].strip()
+            specvalue = line[eqpos+1:]
+            header_data.specialization_names.append( specname )
+            header_data.specialization_values.append( specvalue )
+            line = fs.readline()
+            header_data.line_offset += 1
+            header_data.vertex_offset = header_data.line_offset
+            continue
+
+        if line.find("#[modes]") != -1:
+            # Nothing really, just skip
+            line = fs.readline()
+            header_data.line_offset += 1
+            header_data.vertex_offset = header_data.line_offset
+            continue
+            
+
+        if line.find("#[specializations]") != -1:
+            header_data.reading = "specializations"
+            line = fs.readline()
+            header_data.line_offset += 1
+            header_data.vertex_offset = header_data.line_offset
+            continue
+
+        if line.find("#[vertex]") != -1:
             header_data.reading = "vertex"
             line = fs.readline()
             header_data.line_offset += 1
             header_data.vertex_offset = header_data.line_offset
             continue
 
-        if line.find("[fragment]") != -1:
+        if line.find("#[fragment]") != -1:
             header_data.reading = "fragment"
             line = fs.readline()
             header_data.line_offset += 1
@@ -58,31 +97,15 @@ def include_file_in_legacygl_header(filename, header_data, depth):
             included_file = os.path.relpath(os.path.dirname(filename) + "/" + includeline)
             if not included_file in header_data.vertex_included_files and header_data.reading == "vertex":
                 header_data.vertex_included_files += [included_file]
-                if include_file_in_legacygl_header(included_file, header_data, depth + 1) is None:
+                if include_file_in_gles3_header(included_file, header_data, depth + 1) is None:
                     print("Error in file '" + filename + "': #include " + includeline + "could not be found!")
             elif not included_file in header_data.fragment_included_files and header_data.reading == "fragment":
                 header_data.fragment_included_files += [included_file]
-                if include_file_in_legacygl_header(included_file, header_data, depth + 1) is None:
+                if include_file_in_gles3_header(included_file, header_data, depth + 1) is None:
                     print("Error in file '" + filename + "': #include " + includeline + "could not be found!")
 
             line = fs.readline()
 
-        if line.find("#ifdef ") != -1:
-            if line.find("#ifdef ") != -1:
-                ifdefline = line.replace("#ifdef ", "").strip()
-
-            if line.find("_EN_") != -1:
-                enumbase = ifdefline[: ifdefline.find("_EN_")]
-                ifdefline = ifdefline.replace("_EN_", "_")
-                line = line.replace("_EN_", "_")
-                if enumbase not in header_data.enums:
-                    header_data.enums[enumbase] = []
-                if ifdefline not in header_data.enums[enumbase]:
-                    header_data.enums[enumbase].append(ifdefline)
-
-            elif not ifdefline in header_data.conditionals:
-                header_data.conditionals += [ifdefline]
-
         if line.find("uniform") != -1 and line.lower().find("texunit:") != -1:
             # texture unit
             texunitstr = line[line.find(":") + 1 :].strip()
@@ -144,33 +167,6 @@ def include_file_in_legacygl_header(filename, header_data, depth):
                 if not x in header_data.uniforms:
                     header_data.uniforms += [x]
 
-        if line.strip().find("attribute ") == 0 and line.find("attrib:") != -1:
-            uline = line.replace("in ", "")
-            uline = uline.replace("attribute ", "")
-            uline = uline.replace("highp ", "")
-            uline = uline.replace(";", "")
-            uline = uline[uline.find(" ") :].strip()
-
-            if uline.find("//") != -1:
-                name, bind = uline.split("//")
-                if bind.find("attrib:") != -1:
-                    name = name.strip()
-                    bind = bind.replace("attrib:", "").strip()
-                    header_data.attributes += [(name, bind)]
-
-        if line.strip().find("out ") == 0 and line.find("tfb:") != -1:
-            uline = line.replace("out ", "")
-            uline = uline.replace("highp ", "")
-            uline = uline.replace(";", "")
-            uline = uline[uline.find(" ") :].strip()
-
-            if uline.find("//") != -1:
-                name, bind = uline.split("//")
-                if bind.find("tfb:") != -1:
-                    name = name.strip()
-                    bind = bind.replace("tfb:", "").strip()
-                    header_data.feedbacks += [(name, bind)]
-
         line = line.replace("\r", "")
         line = line.replace("\n", "")
 
@@ -187,12 +183,14 @@ def include_file_in_legacygl_header(filename, header_data, depth):
     return header_data
 
 
-def build_legacygl_header(filename, include, class_suffix, output_attribs):
-    header_data = LegacyGLHeaderStruct()
-    include_file_in_legacygl_header(filename, header_data, 0)
+def build_gles3_header(filename, include, class_suffix, output_attribs):
+    header_data = GLES3HeaderStruct()
+    include_file_in_gles3_header(filename, header_data, 0)
 
     out_file = filename + ".gen.h"
     fd = open(out_file, "w")
+    defspec = 0
+    defvariant = ""
 
     enum_constants = []
 
@@ -202,8 +200,8 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
     out_file_base = out_file_base[out_file_base.rfind("/") + 1 :]
     out_file_base = out_file_base[out_file_base.rfind("\\") + 1 :]
     out_file_ifdef = out_file_base.replace(".", "_").upper()
-    fd.write("#ifndef " + out_file_ifdef + class_suffix + "_120\n")
-    fd.write("#define " + out_file_ifdef + class_suffix + "_120\n")
+    fd.write("#ifndef " + out_file_ifdef + class_suffix + "_GLES3\n")
+    fd.write("#define " + out_file_ifdef + class_suffix + "_GLES3\n")
 
     out_file_class = (
         out_file_base.replace(".glsl.gen.h", "").title().replace("_", "").replace(".", "") + "Shader" + class_suffix
@@ -211,27 +209,42 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
     fd.write("\n\n")
     fd.write('#include "' + include + '"\n\n\n')
     fd.write("class " + out_file_class + " : public Shader" + class_suffix + " {\n\n")
-    fd.write('\t virtual String get_shader_name() const { return "' + out_file_class + '"; }\n')
 
     fd.write("public:\n\n")
 
-    if header_data.conditionals:
-        fd.write("\tenum Conditionals {\n")
-        for x in header_data.conditionals:
-            fd.write("\t\t" + x.upper() + ",\n")
-        fd.write("\t};\n\n")
-
     if header_data.uniforms:
         fd.write("\tenum Uniforms {\n")
         for x in header_data.uniforms:
             fd.write("\t\t" + x.upper() + ",\n")
         fd.write("\t};\n\n")
 
+    if header_data.variant_names:
+        fd.write("\tenum ShaderVariant {\n")
+        for x in header_data.variant_names:
+            fd.write("\t\t" + x + ",\n")
+        fd.write("\t};\n\n")
+    else:
+        fd.write("\tenum ShaderVariant { DEFAULT };\n\n")
+        defvariant="=DEFAULT"
+
+    if header_data.specialization_names:
+        fd.write("\tenum Specializations {\n")
+        counter=0
+        for x in header_data.specialization_names:
+            fd.write("\t\t" + x.upper() + "=" + str(1<<counter)+",\n")            
+            counter+=1
+        fd.write("\t};\n\n")
+        
+    for i in range(len(header_data.specialization_names)):
+            defval = header_data.specialization_values[i].strip()
+            if (defval.upper() == "TRUE" or defval=="1"):
+                defspec|=(1<<i)
+
     fd.write("\t_FORCE_INLINE_ int get_uniform(Uniforms p_uniform) const { return _get_uniform(p_uniform); }\n\n")
-    if header_data.conditionals:
-        fd.write(
-            "\t_FORCE_INLINE_ void set_conditional(Conditionals p_conditional,bool p_enable)  {  _set_conditional(p_conditional,p_enable); }\n\n"
-        )
+    
+    fd.write(
+        "\t_FORCE_INLINE_ void version_bind_shader(RID p_version,ShaderVariant p_variant"+defvariant+",uint64_t p_specialization="+str(defspec)+") { _version_bind_shader(p_version,p_variant,p_specialization); }\n\n"
+    )
     fd.write("\t#ifdef DEBUG_ENABLED\n ")
     fd.write(
         "\t#define _FU if (get_uniform(p_uniform)<0) return; if (!is_version_valid()) return; ERR_FAIL_COND( get_active()!=this ); \n\n "
@@ -368,64 +381,9 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
 
     fd.write("\n\n#undef _FU\n\n\n")
 
-    fd.write("\tvirtual void init() {\n\n")
+    fd.write("protected:\n\n")
 
-    enum_value_count = 0
-
-    if header_data.enums:
-
-        fd.write("\t\t//Written using math, given nonstandarity of 64 bits integer constants..\n")
-        fd.write("\t\tstatic const Enum _enums[]={\n")
-
-        bitofs = len(header_data.conditionals)
-        enum_vals = []
-
-        for xv in header_data.enums:
-            x = header_data.enums[xv]
-            bits = 1
-            amt = len(x)
-            while 2 ** bits < amt:
-                bits += 1
-            strs = "{"
-            for i in range(amt):
-                strs += '"#define ' + x[i] + '\\n",'
-
-                c = {}
-                c["set_mask"] = "uint64_t(" + str(i) + ")<<" + str(bitofs)
-                c["clear_mask"] = (
-                    "((uint64_t(1)<<40)-1) ^ (((uint64_t(1)<<" + str(bits) + ") - 1)<<" + str(bitofs) + ")"
-                )
-                enum_vals.append(c)
-                enum_constants.append(x[i])
-
-            strs += "NULL}"
-
-            fd.write(
-                "\t\t\t{(uint64_t(1<<" + str(bits) + ")-1)<<" + str(bitofs) + "," + str(bitofs) + "," + strs + "},\n"
-            )
-            bitofs += bits
-
-        fd.write("\t\t};\n\n")
-
-        fd.write("\t\tstatic const EnumValue _enum_values[]={\n")
-
-        enum_value_count = len(enum_vals)
-        for x in enum_vals:
-            fd.write("\t\t\t{" + x["set_mask"] + "," + x["clear_mask"] + "},\n")
-
-        fd.write("\t\t};\n\n")
-
-    conditionals_found = []
-    if header_data.conditionals:
-
-        fd.write("\t\tstatic const char* _conditional_strings[]={\n")
-        if header_data.conditionals:
-            for x in header_data.conditionals:
-                fd.write('\t\t\t"#define ' + x + '\\n",\n')
-                conditionals_found.append(x)
-        fd.write("\t\t};\n\n")
-    else:
-        fd.write("\t\tstatic const char **_conditional_strings=NULL;\n")
+    fd.write("\tvirtual void _init() override {\n\n")
 
     if header_data.uniforms:
 
@@ -435,19 +393,18 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
                 fd.write('\t\t\t"' + x + '",\n')
         fd.write("\t\t};\n\n")
     else:
-        fd.write("\t\tstatic const char **_uniform_strings=NULL;\n")
+        fd.write("\t\tstatic const char **_uniform_strings=nullptr;\n")
 
-    if output_attribs:
-        if header_data.attributes:
+    variant_count = 1
+    if len(header_data.variant_defines)>0:
 
-            fd.write("\t\tstatic AttributePair _attribute_pairs[]={\n")
-            for x in header_data.attributes:
-                fd.write('\t\t\t{"' + x[0] + '",' + x[1] + "},\n")
-            fd.write("\t\t};\n\n")
-        else:
-            fd.write("\t\tstatic AttributePair *_attribute_pairs=NULL;\n")
-
-    feedback_count = 0
+        fd.write("\t\tstatic const char* _variant_defines[]={\n")
+        for x in header_data.variant_defines:
+            fd.write('\t\t\t"' + x + '",\n')
+        fd.write("\t\t};\n\n")
+        variant_count=len(header_data.variant_defines)
+    else:
+        fd.write("\t\tstatic const char **_variant_defines[]={""};\n")
 
     if header_data.texunits:
         fd.write("\t\tstatic TexUnitPair _texunit_pairs[]={\n")
@@ -455,7 +412,29 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
             fd.write('\t\t\t{"' + x[0] + '",' + x[1] + "},\n")
         fd.write("\t\t};\n\n")
     else:
-        fd.write("\t\tstatic TexUnitPair *_texunit_pairs=NULL;\n")
+        fd.write("\t\tstatic TexUnitPair *_texunit_pairs=nullptr;\n")
+
+    if header_data.ubos:
+        fd.write("\t\tstatic UBOPair _ubo_pairs[]={\n")
+        for x in header_data.ubos:
+            fd.write('\t\t\t{"' + x[0] + '",' + x[1] + "},\n")
+        fd.write("\t\t};\n\n")
+    else:
+        fd.write("\t\tstatic UBOPair *_ubo_pairs=nullptr;\n")
+
+    if header_data.specialization_names:
+        fd.write("\t\tstatic Specialization _spec_pairs[]={\n")
+        for i in range(len(header_data.specialization_names)):
+            defval = header_data.specialization_values[i].strip()
+            if (defval.upper() == "TRUE" or defval=="1"):
+                defal="true"
+            else:
+                defval="false"
+                
+            fd.write('\t\t\t{"' + header_data.specialization_names[i] + '",' + defval + "},\n")
+        fd.write("\t\t};\n\n")
+    else:
+        fd.write("\t\tstatic Specialization *_spec_pairs=nullptr;\n")
 
     fd.write("\t\tstatic const char _vertex_code[]={\n")
     for x in header_data.vertex_lines:
@@ -465,8 +444,6 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
         fd.write(str(ord("\n")) + ",")
     fd.write("\t\t0};\n\n")
 
-    fd.write("\t\tstatic const int _vertex_code_start=" + str(header_data.vertex_offset) + ";\n")
-
     fd.write("\t\tstatic const char _fragment_code[]={\n")
     for x in header_data.fragment_lines:
         for c in x:
@@ -475,44 +452,23 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
         fd.write(str(ord("\n")) + ",")
     fd.write("\t\t0};\n\n")
 
-    fd.write("\t\tstatic const int _fragment_code_start=" + str(header_data.fragment_offset) + ";\n")
-
-    if output_attribs:
-        fd.write(
-            "\t\tsetup(_conditional_strings,"
-            + str(len(header_data.conditionals))
-            + ",_uniform_strings,"
-            + str(len(header_data.uniforms))
-            + ",_attribute_pairs,"
-            + str(len(header_data.attributes))
-            + ", _texunit_pairs,"
-            + str(len(header_data.texunits))
-            + ",_vertex_code,_fragment_code,_vertex_code_start,_fragment_code_start);\n"
-        )
-    else:
-        fd.write(
-            "\t\tsetup(_conditional_strings,"
-            + str(len(header_data.conditionals))
-            + ",_uniform_strings,"
-            + str(len(header_data.uniforms))
-            + ",_texunit_pairs,"
-            + str(len(header_data.texunits))
-            + ",_enums,"
-            + str(len(header_data.enums))
-            + ",_enum_values,"
-            + str(enum_value_count)
-            + ",_vertex_code,_fragment_code,_vertex_code_start,_fragment_code_start);\n"
-        )
+    fd.write(
+        "\t\tsetup(_vertex_code,_fragment_code,\""
+        + out_file_class + "\","
+        + str(len(header_data.uniforms))
+        + ",_uniform_strings,"
+        + str(len(header_data.ubos))
+        + ",_ubo_pairs,"
+        + str(len(header_data.texunits))
+        + ",_texunit_pairs,"
+        + str(len(header_data.specialization_names))
+        + ",_spec_pairs,"
+        + str(variant_count)
+        + ",_variant_defines);\n"
+    )
 
     fd.write("\t}\n\n")
 
-    if enum_constants:
-
-        fd.write("\tenum EnumConditionals {\n")
-        for x in enum_constants:
-            fd.write("\t\t" + x.upper() + ",\n")
-        fd.write("\t};\n\n")
-        fd.write("\tvoid set_enum_conditional(EnumConditionals p_cond) { _set_enum_conditional(p_cond); }\n")
 
     fd.write("};\n\n")
     fd.write("#endif\n\n")
@@ -521,7 +477,7 @@ def build_legacygl_header(filename, include, class_suffix, output_attribs):
 
 def build_gles3_headers(target, source, env):
     for x in source:
-        build_legacygl_header(str(x), include="drivers/gles3/shader_gles3.h", class_suffix="GLES3", output_attribs=True)
+        build_gles3_header(str(x), include="drivers/gles3/shader_gles3.h", class_suffix="GLES3", output_attribs=True)
 
 
 if __name__ == "__main__":