From fc0db75ac2451a682190bac14782e4b4377f195a Mon Sep 17 00:00:00 2001 From: nic-chen Date: Wed, 3 Feb 2021 22:20:53 +0800 Subject: [PATCH] fix: add defer recover for go routines to prevent abnormal crash --- api/internal/core/storage/etcd.go | 2 + api/internal/core/store/store.go | 2 + api/internal/utils/runtime/runtime.go | 59 ++++++++++++++++++++++ api/internal/utils/runtime/runtime_test.go | 54 ++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 api/internal/utils/runtime/runtime.go create mode 100644 api/internal/utils/runtime/runtime_test.go diff --git a/api/internal/core/storage/etcd.go b/api/internal/core/storage/etcd.go index 136619b6ad..19d8d3dd69 100644 --- a/api/internal/core/storage/etcd.go +++ b/api/internal/core/storage/etcd.go @@ -26,6 +26,7 @@ import ( "github.com/apisix/manager-api/internal/conf" "github.com/apisix/manager-api/internal/log" "github.com/apisix/manager-api/internal/utils" + "github.com/apisix/manager-api/internal/utils/runtime" ) const ( @@ -190,6 +191,7 @@ func (s *EtcdV3Storage) Watch(ctx context.Context, key string) <-chan WatchRespo eventChan := s.client.Watch(ctx, key, clientv3.WithPrefix()) ch := make(chan WatchResponse, 1) go func() { + defer runtime.HandlePanic() for event := range eventChan { output := WatchResponse{ Canceled: event.Canceled, diff --git a/api/internal/core/store/store.go b/api/internal/core/store/store.go index 2b0031cc42..ce1c107552 100644 --- a/api/internal/core/store/store.go +++ b/api/internal/core/store/store.go @@ -32,6 +32,7 @@ import ( "github.com/apisix/manager-api/internal/core/storage" "github.com/apisix/manager-api/internal/log" "github.com/apisix/manager-api/internal/utils" + "github.com/apisix/manager-api/internal/utils/runtime" ) type Interface interface { @@ -108,6 +109,7 @@ func (s *GenericStore) Init() error { c, cancel := context.WithCancel(context.TODO()) ch := s.Stg.Watch(c, s.opt.BasePath) go func() { + defer runtime.HandlePanic() for event := range ch { if event.Canceled { log.Warnf("watch failed: %s", event.Error) diff --git a/api/internal/utils/runtime/runtime.go b/api/internal/utils/runtime/runtime.go new file mode 100644 index 0000000000..7b6dcd99b4 --- /dev/null +++ b/api/internal/utils/runtime/runtime.go @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package runtime + +import ( + "net/http" + "runtime" + + "github.com/apisix/manager-api/internal/log" +) + +var ( + ActuallyPanic = true +) + +var PanicHandlers = []func(interface{}){logPanic} + +func HandlePanic(additionalHandlers ...func(interface{})) { + if err := recover(); err != nil { + for _, fn := range PanicHandlers { + fn(err) + } + for _, fn := range additionalHandlers { + fn(err) + } + if ActuallyPanic { + panic(err) + } + } +} + +func logPanic(r interface{}) { + if r == http.ErrAbortHandler { + return + } + + const size = 32 << 10 + stacktrace := make([]byte, size) + stacktrace = stacktrace[:runtime.Stack(stacktrace, false)] + if _, ok := r.(string); ok { + log.Errorf("observed a panic: %s\n%s", r, stacktrace) + } else { + log.Errorf("observed a panic: %#v (%v)\n%s", r, r, stacktrace) + } +} diff --git a/api/internal/utils/runtime/runtime_test.go b/api/internal/utils/runtime/runtime_test.go new file mode 100644 index 0000000000..103e59f0ad --- /dev/null +++ b/api/internal/utils/runtime/runtime_test.go @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package runtime + +import ( + "testing" +) + +func TestHandleCrash(t *testing.T) { + defer func() { + if x := recover(); x == nil { + t.Errorf("Expected a panic to recover from") + } + }() + defer HandlePanic() + panic("Test Panic") +} + +func TestCustomHandleCrash(t *testing.T) { + old := PanicHandlers + defer func() { PanicHandlers = old }() + var result interface{} + PanicHandlers = []func(interface{}){ + func(r interface{}) { + result = r + }, + } + func() { + defer func() { + if x := recover(); x == nil { + t.Errorf("Expected a panic to recover from") + } + }() + defer HandlePanic() + panic("test") + }() + if result != "test" { + t.Errorf("did not receive custom handler") + } +}