Ilspy upgrade

.github/workflows/all_builds.yml

@@ -28,7 +28,7 @@ env:
   # TODO: change this back to godotengine/godot and target master when #109685 and #109475 are merged
   GODOT_REPOSITORY: nikitalita/godot
   # Change the README too
-  GODOT_MAIN_SYNC_REF: gdre-wb-63227bb
+  GODOT_MAIN_SYNC_REF: gdre-wb-1559ab34c6
   SCONSFLAGS: verbose=yes warnings=all werror=no module_text_server_fb_enabled=yes minizip=yes deprecated=yes
   SCONSFLAGS_TEMPLATE: no_editor_splash=yes module_camera_enabled=no module_mobile_vr_enabled=no module_upnp_enabled=no module_websocket_enabled=no module_csg_enabled=yes module_gridmap_enabled=yes use_static_cpp=yes builtin_freetype=yes builtin_libpng=yes builtin_zlib=yes builtin_libwebp=yes builtin_libvorbis=yes builtin_libogg=yes disable_3d=no
   SCONS_CACHE_MSVC_CONFIG: true
@@ -224,7 +224,7 @@ jobs:
       - name: setup-dotnet
         uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: 9.0.x
+          dotnet-version: 10.0.x
 
       - name: Download pre-built Android Swappy Frame Pacing Library
         if: matrix.platform == 'android'

.gitignore

@@ -44,7 +44,6 @@ gmon.out
 *.swp
 
 # QT project files
-*.config
 *.creator
 *.creator.*
 *.files
@@ -76,7 +75,6 @@ logs/
 *.suo
 *.user
 *.sln.docstates
-*.sln
 *.vcxproj*
 
 # Build results
@@ -323,4 +321,4 @@ platform/windows/godot_res.res
 standalone/spriteTest.*
 standalone/character*
 .tmpcache/
-external_install/
+external_install/

.scripts/rebase_godot.sh

@@ -1,21 +1,30 @@
 #!/bin/sh
 
 SCRIPT_PATH=$(dirname "$0")
+GDRE_PATH=$(realpath "$SCRIPT_PATH/..")
 cd "$SCRIPT_PATH/../../.."
 
 git fetch --all
 git checkout master
 git pull
 
 HEAD=$(git rev-parse HEAD)
+SHORT_HASH=$(git rev-parse --short HEAD)
+NEW_BRANCH_NAME="gdre-wb-$SHORT_HASH"
 
 # check for the existence of the 'nikitalita' remote
 if ! git remote | grep -q "nikitalita"; then
     git remote add nikitalita https://github.com/nikitalita/godot.git
 	git fetch nikitalita
 fi
 
-git checkout working-branch
+# check if the branch already exists
+if git branch -a | grep -q $NEW_BRANCH_NAME; then
+    git branch -D $NEW_BRANCH_NAME
+fi
+git branch -C $NEW_BRANCH_NAME
+
+git checkout $NEW_BRANCH_NAME
 git reset --hard $HEAD
 
 BRANCHES_TO_MERGE=(
@@ -26,9 +35,29 @@ BRANCHES_TO_MERGE=(
 	convert-3.x-escn
 	fix-svg
 	fix-compat-array-shapes
+	gltf-buffer-view-encode-fix
 )
 
+# set fail on error
 for branch in "${BRANCHES_TO_MERGE[@]}"; do
     # merge branch, use a merge commit
     git merge nikitalita/$branch -m "Merge branch '$branch'"
+	if [ $? -ne 0 ]; then
+		echo "Error: Failed to merge branch '$branch'"
+		exit 1
+	fi
 done
+
+# detect OS for cross-platform sed compatibility
+# macOS (BSD sed) requires -i '' while Linux (GNU sed) uses -i
+if [ "$(uname)" = "Darwin" ]; then
+    sed_in_place() { sed -i '' "$@"; }
+else
+    sed_in_place() { sed -i "$@"; }
+fi
+
+git push nikitalita $NEW_BRANCH_NAME --set-upstream
+
+# change the branch name in .github/workflows/all_builds.yml and the README.md
+sed_in_place "s/GODOT_MAIN_SYNC_REF: .*/GODOT_MAIN_SYNC_REF: $NEW_BRANCH_NAME/" "$GDRE_PATH/.github/workflows/all_builds.yml"
+sed_in_place "s/ @ branch \`.*\`/ @ branch \`$NEW_BRANCH_NAME\`/" "$GDRE_PATH/README.md"

README.md

@@ -149,7 +149,7 @@ Note: Make sure to build the editor build first, and to launch the editor to edi
 
 ### Requirements
 
-[Our fork of godot](https://github.com/nikitalita/godot/tree/working-branch) @ `gdre-wb-63227bb`
+[Our fork of godot](https://github.com/nikitalita/godot) @ branch `gdre-wb-1559ab34c6`
 
 - Support for building on 3.x has been dropped and no new features are being pushed
   - Godot RE Tools still retains the ability to decompile 3.x and 2.x projects, however.

exporters/scene_exporter.cpp

@@ -29,6 +29,7 @@
 #include "utility/gdre_logger.h"
 #include "utility/gdre_settings.h"
 
+#include "core/crypto/crypto_core.h"
 #include "core/error/error_list.h"
 #include "core/error/error_macros.h"
 #include "main/main.h"
@@ -38,8 +39,12 @@
 #include "utility/task_manager.h"
 
 #ifndef MAKE_GLTF_COPY
+#ifdef DEBUG_ENABLED
+#define MAKE_GLTF_COPY 1
+#else
 #define MAKE_GLTF_COPY 0
 #endif
+#endif
 #ifndef PRINT_PERF_CSV
 #define PRINT_PERF_CSV 0
 #endif
@@ -2408,6 +2413,85 @@ void GLTFDocumentExtensionPhysicsRemover::convert_scene_node(Ref<GLTFState> p_st
 	}
 }
 
+Ref<Image> GLBExporterInstance::_parse_image_bytes_into_image(const Ref<GLTFState> &p_state, const Vector<uint8_t> &p_bytes, const String &p_mime_type, int p_index, String &r_file_extension) {
+	Ref<Image> r_image;
+	r_image.instantiate();
+	// Check if any GLTFDocumentExtensions want to import this data as an image.
+	for (auto &ext : GLTFDocument::get_all_gltf_document_extensions()) {
+		ERR_CONTINUE(ext.is_null());
+		Error err = ext->parse_image_data(p_state, p_bytes, p_mime_type, r_image);
+		ERR_CONTINUE_MSG(err != OK, "glTF: Encountered error " + itos(err) + " when parsing image " + itos(p_index) + " in file " + p_state->get_filename() + ". Continuing.");
+		if (!r_image->is_empty()) {
+			r_file_extension = ext->get_image_file_extension();
+			return r_image;
+		}
+	}
+	// If no extension wanted to import this data as an image, try to load a PNG or JPEG.
+	// First we honor the mime types if they were defined.
+	if (p_mime_type == "image/png") { // Load buffer as PNG.
+		r_image->load_png_from_buffer(p_bytes);
+		r_file_extension = ".png";
+	} else if (p_mime_type == "image/jpeg") { // Loader buffer as JPEG.
+		r_image->load_jpg_from_buffer(p_bytes);
+		r_file_extension = ".jpg";
+	}
+	// If we didn't pass the above tests, we attempt loading as PNG and then JPEG directly.
+	// This covers URIs with base64-encoded data with application/* type but
+	// no optional mimeType property, or bufferViews with a bogus mimeType
+	// (e.g. `image/jpeg` but the data is actually PNG).
+	// That's not *exactly* what the spec mandates but this lets us be
+	// lenient with bogus glb files which do exist in production.
+	if (r_image->is_empty()) { // Try PNG first.
+		r_image->load_png_from_buffer(p_bytes);
+	}
+	if (r_image->is_empty()) { // And then JPEG.
+		r_image->load_jpg_from_buffer(p_bytes);
+	}
+	// If it still can't be loaded, give up and insert an empty image as placeholder.
+	if (r_image->is_empty()) {
+		ERR_PRINT(vformat("glTF: Couldn't load image index '%d' with its given mimetype: %s.", p_index, p_mime_type));
+	}
+	return r_image;
+}
+
+String GLBExporterInstance::get_gltf_image_hash(const Ref<GLTFState> &p_state, const Dictionary &dict, int p_index) {
+	String image_hash;
+	String mime_type;
+	if (dict.has("mimeType")) { // Should be "image/png", "image/jpeg", or something handled by an extension.
+		mime_type = dict["mimeType"];
+	}
+
+	String resource_uri;
+	// We only need the image hash if it's embedded in the glTF file, so if it's not a bufferView, we can return early.
+	Vector<uint8_t> data;
+	if (dict.has("bufferView")) {
+		// Handles the third bullet point from the spec (bufferView).
+		ERR_FAIL_COND_V_MSG(mime_type.is_empty(), "", vformat("glTF: Image index '%d' specifies 'bufferView' but no 'mimeType', which is invalid.", p_index));
+		const GLTFBufferViewIndex bvi = dict["bufferView"];
+		auto buffer_views = p_state->get_buffer_views();
+		ERR_FAIL_INDEX_V(bvi, buffer_views.size(), "");
+		Ref<GLTFBufferView> bv = buffer_views[bvi];
+		const GLTFBufferIndex bi = bv->get_buffer();
+		auto buffers = p_state->get_buffers();
+		ERR_FAIL_INDEX_V(bi, buffers.size(), "");
+		ERR_FAIL_COND_V(bv->get_byte_offset() + bv->get_byte_length() > buffers[bi].size(), "");
+		const PackedByteArray &buffer = buffers[bi];
+		data = buffer.slice(bv->get_byte_offset(), bv->get_byte_offset() + bv->get_byte_length());
+	}
+	if (data.is_empty()) {
+		return image_hash;
+	}
+	String ext;
+	Ref<Image> img = _parse_image_bytes_into_image(p_state, data, mime_type, p_index, ext);
+	if (img.is_valid()) {
+		auto img_data = img->get_data();
+		unsigned char md5_hash[16];
+		CryptoCore::md5(img_data.ptr(), img_data.size(), md5_hash);
+		image_hash = String::hex_encode_buffer(md5_hash, 16);
+	}
+	return image_hash;
+}
+
 Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_dest_path) {
 	{
 		GDRE_SCN_EXP_CHECK_CANCEL();
@@ -2564,7 +2648,9 @@ Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_d
 				} else {
 					name = path.get_file().get_basename();
 					external_deps_updated.insert(path);
-					image_path_to_data_hash[path] = FileAccess::get_md5(path);
+					if (updating_import_info && is_batch_export) {
+						image_path_to_data_hash[path] = get_gltf_image_hash(state, image_dict, i);
+					}
 				}
 				check_unique(name, image_map);
 
@@ -3008,7 +3094,9 @@ void GLBExporterInstance::_update_import_params(const String &p_dest_path) {
 
 	iinfo->set_param("_subresources", _subresources_dict);
 	Dictionary extra_info = report->get_extra_info();
-	extra_info["image_path_to_data_hash"] = image_path_to_data_hash;
+	if (!image_path_to_data_hash.is_empty()) {
+		extra_info["image_path_to_data_hash"] = image_path_to_data_hash;
+	}
 	report->set_extra_info(extra_info);
 }
 
@@ -3447,18 +3535,17 @@ struct BatchExportToken : public TaskRunnerStruct {
 		if (check_unsupported()) {
 			err = ERR_UNAVAILABLE;
 			_set_unsupported(report, ver_major, is_obj_output());
-			preload_done = true;
+			after_preload();
 			return false;
 		}
 
 		if (is_text_output()) {
-			preload_done = true;
 			return false;
 		}
 		err = _check_cancelled();
 		if (err != OK) {
 			report->set_error(err);
-			preload_done = true;
+			after_preload();
 			return false;
 		}
 		instance._initial_set(p_src_path, report);
@@ -3474,7 +3561,7 @@ struct BatchExportToken : public TaskRunnerStruct {
 			}
 			if (err != OK) {
 				report->set_error(err);
-				preload_done = true;
+				after_preload();
 				return false;
 			}
 			_scene = scene;
@@ -3484,7 +3571,7 @@ struct BatchExportToken : public TaskRunnerStruct {
 					_scene = nullptr;
 					err = ERR_CANT_ACQUIRE_RESOURCE;
 					report->set_error(err);
-					preload_done = true;
+					after_preload();
 					return false;
 				}
 				root = instance._set_stuff_from_instanced_scene(root);
@@ -3501,10 +3588,19 @@ struct BatchExportToken : public TaskRunnerStruct {
 			surface_count = get_total_surface_count(meshes);
 		}
 		// print_line("Preloaded scene " + p_src_path);
-		preload_done = true;
+		after_preload();
 		return do_on_main_thread;
 	}
 
+	void after_preload() {
+		if (Thread::is_main_thread() && _scene.is_valid()) {
+			// We have to flush the message queue after the scene is loaded;
+			// Certain resources like NoiseTexture2D can queue up deferred calls that will cause crashes if not flushed before the scene is manipulated or freed
+			GDRESettings::main_iteration();
+		}
+		preload_done = true;
+	}
+
 	void batch_export_instanced_scene() {
 		while (!preload_done && _check_cancelled() == OK) {
 			OS::get_singleton()->delay_usec(10000);

exporters/scene_exporter.h

@@ -198,6 +198,9 @@ class GLBExporterInstance {
 
 	String demangle_name(const String &obj_name);
 
+	static Ref<Image> _parse_image_bytes_into_image(const Ref<GLTFState> &p_state, const Vector<uint8_t> &p_bytes, const String &p_mime_type, int p_index, String &r_file_extension);
+	static String get_gltf_image_hash(const Ref<GLTFState> &p_state, const Dictionary &dict, int p_index);
+
 public:
 	GLBExporterInstance(String p_output_dir, Dictionary curr_options = {}, bool p_project_recovery = false);
 

godot-mono-decomp/GodotMonoDecomp/Common.cs

@@ -31,7 +31,7 @@
 		// .NET Framework 	all 	  C# 7.3
 
 		// determine the dotnet version
 		var dotnetVersion = TargetServices.DetectTargetFramework(module);
 		if (dotnetVersion == null)
 		{
 			return LanguageVersion.CSharp7_3;
@@ -246,7 +246,7 @@
 			.Select(field => field.Name).ToArray();
 	}
 
-	public static string GetEnumValueName(IType type, int value, string defaultValue = "")
+	public static string GetEnumValueName(IType type, long value, string defaultValue = "")
 	{
 		var names = GetEnumValueNames(type);
 		if (names.Length == 0 || value >= names.Length)
@@ -255,4 +255,28 @@
 		}
 		return names[value];
 	}
+
+	public static Guid GenerateDeterministicGuidFromString(string input)
+	{
+		const byte Variant10xxMask = 0xC0;
+		const byte Variant10xxValue = 0x80;
+
+		const ushort VersionMask = 0xF000;
+		const ushort Version4Value = 0x4000;
+		using (var md5 = System.Security.Cryptography.MD5.Create())
+		{
+			byte[] inputBytes = Encoding.UTF8.GetBytes(input);
+			byte[] hashBytes = md5.ComputeHash(inputBytes);
+			short _c = (short)(hashBytes[3] | hashBytes[4] << 8);
+			byte _d = hashBytes[5];
+			_c = (short)((_c & ~VersionMask) | Version4Value);
+			_d = (byte)((_d & ~Variant10xxMask) | Variant10xxValue);
+
+			hashBytes[3] = (byte)(_c & 0xFF);
+			hashBytes[4] = (byte)((_c >> 8) & 0xFF);
+			hashBytes[5] = _d;
+
+			return new Guid(hashBytes);
+		}
+	}
 }

godot-mono-decomp/GodotMonoDecomp/GodotExpressionTokenWriter.cs

@@ -36,17 +36,21 @@ public class GodotExpressionOutputVisitor : CSharpOutputVisitor
 	TextWriter tw;
 	private bool isDict = false;
 
-	public GodotExpressionOutputVisitor(TextWriter w, CSharpFormattingOptions formattingOptions)
+	private GodotProjectDecompiler? godotDecompiler;
+
+	public GodotExpressionOutputVisitor(TextWriter w, CSharpFormattingOptions formattingOptions, GodotProjectDecompiler? godotDecompiler = null)
 		: base(new GodotExpressionTokenWriter(w), formattingOptions)
 	{
 		tw = w;
+		this.godotDecompiler = godotDecompiler;
 	}
 
 
 
-	public GodotExpressionOutputVisitor(TextWriter w)
-		: this(w, SingleLineFormattingOptions)
-	{}
+	public GodotExpressionOutputVisitor(TextWriter w, GodotProjectDecompiler? godotDecompiler = null)
+		: this(w, SingleLineFormattingOptions, godotDecompiler)
+	{
+	}
 
 	public static readonly CSharpFormattingOptions SingleLineFormattingOptions = GetSingleLineFormattingOptions();
 
@@ -231,10 +235,10 @@ private static CSharpFormattingOptions GetSingleLineFormattingOptions()
 		return settings.CSharpFormattingOptions;
 	}
 
-	public static string GetString(AstNode node)
+	public static string GetString(AstNode node, GodotProjectDecompiler? godotDecompiler = null)
 	{
 		var stringWriter = new StringWriter();
-		var visitor = new GodotExpressionOutputVisitor(stringWriter);
+		var visitor = new GodotExpressionOutputVisitor(stringWriter, godotDecompiler);
 		node.AcceptVisitor(visitor);
 		return stringWriter.ToString();
 	}
@@ -501,7 +505,7 @@ bool IsComplexExpression(Expression ex)
 	}
 	public override void VisitMemberReferenceExpression(MemberReferenceExpression memberReferenceExpression)
 	{
-		string? text = GodotStuff.ReplaceMemberReference(memberReferenceExpression);
+		string? text = GodotStuff.ReplaceMemberReference(memberReferenceExpression, godotDecompiler);
 		if (text != null)
 		{
 			StartNode(memberReferenceExpression);