Merge pull request #19629 from kotcrab/struct-viewer

Struct viewer debugging tool

CMakeLists.txt

@@ -884,6 +884,8 @@ add_library(Common STATIC
 	Common/FakeCPUDetect.cpp
 	Common/ExceptionHandlerSetup.cpp
 	Common/ExceptionHandlerSetup.h
+	Common/GhidraClient.h
+	Common/GhidraClient.cpp
 	Common/Log.h
 	Common/Log.cpp
 	Common/Log/ConsoleListener.cpp
@@ -1523,6 +1525,8 @@ list(APPEND NativeAppSource
 	UI/ImDebugger/ImDebugger.h
 	UI/ImDebugger/ImDisasmView.cpp
 	UI/ImDebugger/ImDisasmView.h
+	UI/ImDebugger/ImStructViewer.cpp
+	UI/ImDebugger/ImStructViewer.h
 	UI/DiscordIntegration.cpp
 	UI/NativeApp.cpp
 	UI/BackgroundAudio.h

Common/Common.vcxproj

@@ -591,6 +591,7 @@
     <ClInclude Include="Crypto\sha256.h" />
     <ClInclude Include="DbgNew.h" />
     <ClInclude Include="ExceptionHandlerSetup.h" />
+    <ClInclude Include="GhidraClient.h" />
     <ClInclude Include="GraphicsContext.h" />
     <ClInclude Include="Log.h" />
     <ClInclude Include="Log\LogManager.h" />
@@ -1021,6 +1022,7 @@
     <ClCompile Include="GPU\Vulkan\VulkanRenderManager.cpp" />
     <ClCompile Include="Input\GestureDetector.cpp" />
     <ClCompile Include="Input\InputState.cpp" />
+    <ClCompile Include="GhidraClient.cpp" />
     <ClCompile Include="Log.cpp" />
     <ClCompile Include="Math\curves.cpp" />
     <ClCompile Include="Math\expression_parser.cpp" />

Common/Common.vcxproj.filters

@@ -6,6 +6,7 @@
     <ClInclude Include="CommonFuncs.h" />
     <ClInclude Include="CommonTypes.h" />
     <ClInclude Include="CPUDetect.h" />
+    <ClInclude Include="GhidraClient.h" />
     <ClInclude Include="Log.h" />
     <ClInclude Include="MemArena.h" />
     <ClInclude Include="MemoryUtil.h" />
@@ -705,6 +706,7 @@
       <Filter>Serialize</Filter>
     </ClCompile>
     <ClCompile Include="TimeUtil.cpp" />
+    <ClCompile Include="GhidraClient.cpp" />
     <ClCompile Include="Log.cpp" />
     <ClCompile Include="SysError.cpp" />
     <ClCompile Include="..\ext\libpng17\png.c">

Common/GhidraClient.cpp

@@ -0,0 +1,204 @@
+#include "Common/Data/Format/JSONReader.h"
+#include "Common/Net/HTTPClient.h"
+#include "Common/Thread/ThreadUtil.h"
+
+#include "Common/GhidraClient.h"
+
+using namespace json;
+
+static GhidraTypeKind ResolveTypeKind(const std::string& kind) {
+	if (kind == "ENUM") return ENUM;
+	if (kind == "TYPEDEF") return TYPEDEF;
+	if (kind == "POINTER") return POINTER;
+	if (kind == "ARRAY") return ARRAY;
+	if (kind == "STRUCTURE") return STRUCTURE;
+	if (kind == "UNION") return UNION;
+	if (kind == "FUNCTION_DEFINITION") return FUNCTION_DEFINITION;
+	if (kind == "BUILT_IN") return BUILT_IN;
+	return UNKNOWN;
+}
+
+GhidraClient::~GhidraClient() {
+	if (thread_.joinable()) {
+		thread_.join();
+	}
+}
+
+void GhidraClient::FetchAll(const std::string& host, const int port) {
+	std::lock_guard<std::mutex> lock(mutex_);
+	if (status_ != Status::Idle) {
+		return;
+	}
+	status_ = Status::Pending;
+	thread_ = std::thread([this, host, port] {
+		SetCurrentThreadName("GhidraClient");
+		FetchAllDo(host, port);
+	});
+}
+
+bool GhidraClient::FetchAllDo(const std::string& host, const int port) {
+	std::lock_guard<std::mutex> lock(mutex_);
+	host_ = host;
+	port_ = port;
+	const bool result = FetchTypes() && FetchSymbols();
+	status_ = Status::Ready;
+	return result;
+}
+
+void GhidraClient::UpdateResult() {
+	std::lock_guard<std::mutex> lock(mutex_);
+	if (status_ != Status::Ready) {
+		return;
+	}
+	if (thread_.joinable()) {
+		thread_.join();
+	}
+	result = std::move(pendingResult_);
+	pendingResult_ = Result();
+	status_ = Status::Idle;
+}
+
+bool GhidraClient::FetchSymbols() {
+	std::string json;
+	if (!FetchResource("/v1/symbols", json)) {
+		return false;
+	}
+	JsonReader reader(json.c_str(), json.size());
+	if (!reader.ok()) {
+		pendingResult_.error = "symbols parsing error";
+		return false;
+	}
+	const JsonValue entries = reader.root().getArray("symbols")->value;
+	if (entries.getTag() != JSON_ARRAY) {
+		pendingResult_.error = "symbols is not an array";
+		return false;
+	}
+
+	for (const auto pEntry : entries) {
+		JsonGet entry = pEntry->value;
+
+		GhidraSymbol symbol;
+		symbol.address = entry.getInt("address", 0);
+		symbol.name = entry.getStringOr("name", "");
+		symbol.label = strcmp(entry.getStringOr("type", ""), "Label") == 0;
+		symbol.userDefined = strcmp(entry.getStringOr("source", ""), "USER_DEFINED") == 0;
+		symbol.dataTypePathName = entry.getStringOr("dataTypePathName", "");
+		pendingResult_.symbols.emplace_back(symbol);
+	}
+	return true;
+}
+
+bool GhidraClient::FetchTypes() {
+	std::string json;
+	if (!FetchResource("/v1/types", json)) {
+		return false;
+	}
+	JsonReader reader(json.c_str(), json.size());
+	if (!reader.ok()) {
+		pendingResult_.error = "types parsing error";
+		return false;
+	}
+	const JsonValue entries = reader.root().getArray("types")->value;
+	if (entries.getTag() != JSON_ARRAY) {
+		pendingResult_.error = "types is not an array";
+		return false;
+	}
+
+	for (const auto pEntry : entries) {
+		const JsonGet entry = pEntry->value;
+
+		GhidraType type;
+		type.displayName = entry.getStringOr("displayName", "");
+		type.pathName = entry.getStringOr("pathName", "");
+		type.length = entry.getInt("length", 0);
+		type.alignedLength = entry.getInt("alignedLength", 0);
+		type.zeroLength = entry.getBool("zeroLength", false);
+		type.description = entry.getStringOr("description", "");
+		type.kind = ResolveTypeKind(entry.getStringOr("kind", ""));
+
+		switch (type.kind) {
+			case ENUM: {
+				const JsonNode* enumEntries = entry.getArray("members");
+				if (!enumEntries) {
+					pendingResult_.error = "missing enum members";
+					return false;
+				}
+				for (const JsonNode* pEnumEntry : enumEntries->value) {
+					JsonGet enumEntry = pEnumEntry->value;
+					GhidraEnumMember member;
+					member.name = enumEntry.getStringOr("name", "");
+					member.value = enumEntry.getInt("value", 0);
+					member.comment = enumEntry.getStringOr("comment", "");
+					type.enumMembers.push_back(member);
+				}
+				break;
+			}
+			case TYPEDEF:
+				type.typedefTypePathName = entry.getStringOr("typePathName", "");
+				type.typedefBaseTypePathName = entry.getStringOr("baseTypePathName", "");
+				break;
+			case POINTER:
+				type.pointerTypePathName = entry.getStringOr("typePathName", "");
+				break;
+			case ARRAY:
+				type.arrayTypePathName = entry.getStringOr("typePathName", "");
+				type.arrayElementLength = entry.getInt("elementLength", 0);
+				type.arrayElementCount = entry.getInt("elementCount", 0);
+				break;
+			case STRUCTURE:
+			case UNION: {
+				const JsonNode* compositeEntries = entry.getArray("members");
+				if (!compositeEntries) {
+					pendingResult_.error = "missing composite members";
+					return false;
+				}
+				for (const JsonNode* pCompositeEntry : compositeEntries->value) {
+					JsonGet compositeEntry = pCompositeEntry->value;
+					GhidraCompositeMember member;
+					member.fieldName = compositeEntry.getStringOr("fieldName", "");
+					member.ordinal = compositeEntry.getInt("ordinal", 0);
+					member.offset = compositeEntry.getInt("offset", 0);
+					member.length = compositeEntry.getInt("length", 0);
+					member.typePathName = compositeEntry.getStringOr("typePathName", "");
+					member.comment = compositeEntry.getStringOr("comment", "");
+					type.compositeMembers.push_back(member);
+				}
+				break;
+			}
+			case FUNCTION_DEFINITION:
+				type.functionPrototypeString = entry.getStringOr("prototypeString", "");
+				break;
+			case BUILT_IN:
+				type.builtInGroup = entry.getStringOr("group", "");
+				break;
+			default:
+				continue;
+		}
+
+		pendingResult_.types.emplace(type.pathName, type);
+	}
+	return true;
+}
+
+bool GhidraClient::FetchResource(const std::string& path, std::string& outResult) {
+	http::Client http;
+	if (!http.Resolve(host_.c_str(), port_)) {
+		pendingResult_.error = "can't resolve host";
+		return false;
+	}
+	bool cancelled = false;
+	if (!http.Connect(1, 5.0, &cancelled)) {
+		pendingResult_.error = "can't connect to host";
+		return false;
+	}
+	net::RequestProgress progress(&cancelled);
+	Buffer result;
+	const int code = http.GET(http::RequestParams(path.c_str()), &result, &progress);
+	http.Disconnect();
+	if (code != 200) {
+		pendingResult_.error = "unsuccessful response code from API endpoint";
+		return false;
+	}
+	result.TakeAll(&outResult);
+	return true;
+}

Common/GhidraClient.h

@@ -0,0 +1,142 @@
+#pragma once
+
+#include <atomic>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+/**
+ * Represents symbol from a Ghidra's program.
+ * A symbol can be for example a data label, instruction label or a function.
+ */
+struct GhidraSymbol {
+	u32 address = 0;
+	std::string name;
+	bool label;
+	bool userDefined;
+	std::string dataTypePathName;
+};
+
+/** Possible kinds of data types, such as enum, structures or built-ins (Ghidra's primitive types). */
+enum GhidraTypeKind {
+	ENUM,
+	TYPEDEF,
+	POINTER,
+	ARRAY,
+	STRUCTURE,
+	UNION,
+	FUNCTION_DEFINITION,
+	BUILT_IN,
+	UNKNOWN,
+};
+
+/** Describes single member of an enum type. */
+struct GhidraEnumMember {
+	std::string name;
+	u64 value = 0;
+	std::string comment;
+};
+
+/** Describes single member of a composite (structure or union) type. */
+struct GhidraCompositeMember {
+	std::string fieldName;
+	u32 ordinal = 0;
+	u32 offset = 0;
+	int length = 0;
+	std::string typePathName;
+	std::string comment;
+};
+
+/**
+ * Describes data type from Ghidra. Note that some fields of this structure will only be populated depending on the
+ * type's kind. Each type has a display name that is suitable for displaying to the user and a path name that
+ * unambiguously identifies this type.
+ */
+struct GhidraType {
+	GhidraTypeKind kind;
+	std::string displayName;
+	std::string pathName;
+	int length = 0;
+	int alignedLength = 0;
+	bool zeroLength = false;
+	std::string description;
+
+	std::vector<GhidraCompositeMember> compositeMembers;
+	std::vector<GhidraEnumMember> enumMembers;
+	std::string pointerTypePathName;
+	std::string typedefTypePathName;
+	std::string typedefBaseTypePathName;
+	std::string arrayTypePathName;
+	int arrayElementLength = 0;
+	u32 arrayElementCount = 0;
+	std::string functionPrototypeString;
+	std::string builtInGroup;
+};
+
+/**
+ * GhidraClient implements fetching data (such as symbols or types) from a remote Ghidra project.
+ *
+ * This client uses unofficial API provided by the third party "ghidra-rest-api" extension. The extension is
+ * available at https://github.com/kotcrab/ghidra-rest-api.
+ *
+ * This class doesn't fetch data from every possible endpoint, only those that are actually used by PPSSPP.
+ *
+ * How to use:
+ * 1. The client is created in the Idle status.
+ * 2. To start fetching data call the FetchAll() method. The client goes to Pending status and the data is fetched
+ *    in a background thread so your code remains unblocked.
+ * 3. Periodically check with the Ready() method is the operation has completed. (i.e. check if the client
+      is in the Ready status)
+ * 4. If the client is ready call UpdateResult() to update result field with new data.
+ * 5. The client is now back to Idle status, and you can call FetchAll() again later if needed.
+ */
+class GhidraClient {
+	enum class Status {
+		Idle,
+		Pending,
+		Ready,
+	};
+
+public:
+	struct Result {
+		std::vector<GhidraSymbol> symbols;
+		std::unordered_map<std::string, GhidraType> types;
+		std::string error;
+	};
+
+	/** Current result of the client. Your thread is safe to access this regardless of client status. */
+	Result result;
+
+	~GhidraClient();
+
+	/** If client is idle then asynchronously starts fetching data from Ghidra. */
+	void FetchAll(const std::string& host, int port);
+
+	/** If client is ready then updates the result field with newly fetched data. This must be called from the thread
+	 * using the result. */
+	void UpdateResult();
+
+	bool Idle() const { return status_ == Status::Idle; }
+
+	bool Ready() const { return status_ == Status::Ready; }
+
+	bool Failed() const { return !result.error.empty(); }
+
+private:
+	std::thread thread_;
+	std::mutex mutex_;
+	std::atomic<Status> status_;
+	Result pendingResult_;
+	std::string host_;
+	int port_ = 0;
+
+	bool FetchAllDo(const std::string& host, int port);
+
+	bool FetchSymbols();
+
+	bool FetchTypes();
+
+	bool FetchResource(const std::string& path, std::string& outResult);
+};