fix: add defer recover for go routines to prevent abnormal crash

api/internal/core/storage/etcd.go

@@ -26,6 +26,7 @@ import (
 	"github.com/apisix/manager-api/internal/conf"
 	"github.com/apisix/manager-api/internal/log"
 	"github.com/apisix/manager-api/internal/utils"
+	"github.com/apisix/manager-api/internal/utils/runtime"
 )
 
 const (
@@ -190,6 +191,7 @@ func (s *EtcdV3Storage) Watch(ctx context.Context, key string) <-chan WatchRespo
 	eventChan := s.client.Watch(ctx, key, clientv3.WithPrefix())
 	ch := make(chan WatchResponse, 1)
 	go func() {
+		defer runtime.HandlePanic()
 		for event := range eventChan {
 			output := WatchResponse{
 				Canceled: event.Canceled,

api/internal/core/store/store.go

@@ -32,6 +32,7 @@ import (
 	"github.com/apisix/manager-api/internal/core/storage"
 	"github.com/apisix/manager-api/internal/log"
 	"github.com/apisix/manager-api/internal/utils"
+	"github.com/apisix/manager-api/internal/utils/runtime"
 )
 
 type Interface interface {
@@ -108,6 +109,7 @@ func (s *GenericStore) Init() error {
 	c, cancel := context.WithCancel(context.TODO())
 	ch := s.Stg.Watch(c, s.opt.BasePath)
 	go func() {
+		defer runtime.HandlePanic()
 		for event := range ch {
 			if event.Canceled {
 				log.Warnf("watch failed: %s", event.Error)

api/internal/utils/runtime/runtime.go

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package runtime
+
+import (
+	"net/http"
+	"runtime"
+
+	"github.com/apisix/manager-api/internal/log"
+)
+
+var (
+	ActuallyPanic = true
+)
+
+var PanicHandlers = []func(interface{}){logPanic}
+
+func HandlePanic(additionalHandlers ...func(interface{})) {
+	if err := recover(); err != nil {
+		for _, fn := range PanicHandlers {
+			fn(err)
+		}
+		for _, fn := range additionalHandlers {
+			fn(err)
+		}
+		if ActuallyPanic {
+			panic(err)
+		}
+	}
+}
+
+func logPanic(r interface{}) {
+	if r == http.ErrAbortHandler {
+		return
+	}
+
+	const size = 32 << 10
+	stacktrace := make([]byte, size)
+	stacktrace = stacktrace[:runtime.Stack(stacktrace, false)]
+	if _, ok := r.(string); ok {
+		log.Errorf("observed a panic: %s\n%s", r, stacktrace)
+	} else {
+		log.Errorf("observed a panic: %#v (%v)\n%s", r, r, stacktrace)
+	}
+}

api/internal/utils/runtime/runtime_test.go

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package runtime
+
+import (
+	"testing"
+)
+
+func TestHandleCrash(t *testing.T) {
+	defer func() {
+		if x := recover(); x == nil {
+			t.Errorf("Expected a panic to recover from")
+		}
+	}()
+	defer HandlePanic()
+	panic("Test Panic")
+}
+
+func TestCustomHandleCrash(t *testing.T) {
+	old := PanicHandlers
+	defer func() { PanicHandlers = old }()
+	var result interface{}
+	PanicHandlers = []func(interface{}){
+		func(r interface{}) {
+			result = r
+		},
+	}
+	func() {
+		defer func() {
+			if x := recover(); x == nil {
+				t.Errorf("Expected a panic to recover from")
+			}
+		}()
+		defer HandlePanic()
+		panic("test")
+	}()
+	if result != "test" {
+		t.Errorf("did not receive custom handler")
+	}
+}