softgpu: Detect binner alloc fail and bail

Core/System.cpp

@@ -487,6 +487,14 @@ bool PSP_InitUpdate(std::string *error_string) {
 	if (pspIsInited) {
 		Core_NotifyLifecycle(CoreLifecycle::START_COMPLETE);
 		pspIsRebooting = false;
+
+		// If GPU init failed during IsReady checks, bail.
+		if (!GPU_IsStarted()) {
+			*error_string = "Unable to initialize rendering engine.";
+			pspIsRebooting = false;
+			PSP_Shutdown();
+			return true;
+		}
 	}
 	return pspIsInited;
 }

GPU/GPU.cpp

@@ -58,6 +58,12 @@ bool GPU_IsReady() {
 	return false;
 }
 
+bool GPU_IsStarted() {
+	if (gpu)
+		return gpu->IsReady() && gpu->IsStarted();
+	return false;
+}
+
 bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw) {
 	const auto &gpuCore = PSP_CoreParameter().gpuCore;
 	_assert_(draw || gpuCore == GPUCORE_SOFTWARE);
@@ -106,7 +112,10 @@ bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw) {
 #endif
 	}
 
-	return gpu != NULL;
+	if (gpu && gpu->IsReady() && !gpu->IsStarted())
+		SetGPU<SoftGPU>(nullptr);
+
+	return gpu != nullptr;
 #endif
 }
 #ifdef USE_CRT_DBG

GPU/GPU.h

@@ -147,4 +147,5 @@ namespace Draw {
 
 bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw);
 bool GPU_IsReady();
+bool GPU_IsStarted();
 void GPU_Shutdown();

GPU/GPUCommon.h

@@ -86,6 +86,9 @@ class GPUCommon : public GPUInterface, public GPUDebugInterface {
 	bool IsReady() override {
 		return true;
 	}
+	bool IsStarted() override {
+		return true;
+	}
 	void CancelReady() override {}
 	void Reinitialize() override;
 

GPU/GPUInterface.h

@@ -191,6 +191,7 @@ class GPUInterface {
 	// Initialization
 	virtual bool IsReady() = 0;
 	virtual void CancelReady() = 0;
+	virtual bool IsStarted() = 0;
 	virtual void InitClear() = 0;
 	virtual void Reinitialize() = 0;
 

GPU/Software/SoftGpu.cpp

@@ -433,11 +433,15 @@ SoftGPU::SoftGPU(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	Rasterizer::Init();
 	Sampler::Init();
 	drawEngine_ = new SoftwareDrawEngine();
+	if (!drawEngine_)
+		return;
+
 	drawEngine_->Init();
 	drawEngineCommon_ = drawEngine_;
 
 	// Push the initial CLUT buffer in case it's all zero (we push only on change.)
-	drawEngine_->transformUnit.NotifyClutUpdate(clut);
+	if (drawEngine_->transformUnit.IsStarted())
+		drawEngine_->transformUnit.NotifyClutUpdate(clut);
 
 	// No need to flush for simple parameter changes.
 	flushOnParams_ = false;
@@ -762,6 +766,10 @@ void SoftGPU::FastRunLoop(DisplayList &list) {
 	dirtyFlags_ = dirty;
 }
 
+bool SoftGPU::IsStarted() {
+	return drawEngine_ && drawEngine_->transformUnit.IsStarted();
+}
+
 void SoftGPU::ExecuteOp(u32 op, u32 diff) {
 	const u8 cmd = op >> 24;
 	const auto info = softgpuCmdInfo[cmd];

GPU/Software/SoftGpu.h

@@ -128,6 +128,7 @@ class SoftGPU : public GPUCommon {
 	~SoftGPU();
 
 	u32 CheckGPUFeatures() const override { return 0; }
+	bool IsStarted() override;
 	void InitClear() override {}
 	void ExecuteOp(u32 op, u32 diff) override;
 	void FinishDeferred() override;

GPU/Software/TransformUnit.cpp

@@ -42,6 +42,8 @@
 
 TransformUnit::TransformUnit() {
 	decoded_ = (u8 *)AllocateMemoryPages(TRANSFORM_BUF_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);
+	if (!decoded_)
+		return;
 	binner_ = new BinManager();
 }
 
@@ -50,6 +52,10 @@ TransformUnit::~TransformUnit() {
 	delete binner_;
 }
 
+bool TransformUnit::IsStarted() {
+	return binner_ && decoded_;
+}
+
 SoftwareDrawEngine::SoftwareDrawEngine() {
 	// All this is a LOT of memory, need to see if we can cut down somehow.  Used for splines.
 	decoded = (u8 *)AllocateMemoryPages(DECODED_VERTEX_BUFFER_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);

GPU/Software/TransformUnit.h

@@ -117,6 +117,8 @@ class TransformUnit {
 	TransformUnit();
 	~TransformUnit();
 
+	bool IsStarted();
+
 	static WorldCoords ModelToWorldNormal(const ModelCoords& coords);
 	static WorldCoords ModelToWorld(const ModelCoords& coords);
 	static ViewCoords WorldToView(const WorldCoords& coords);