From 3850979308406d321ab1db1abea44373bc3275d9 Mon Sep 17 00:00:00 2001 From: dragondriver Date: Mon, 29 Nov 2021 14:29:24 +0800 Subject: [PATCH] Show stack trace info when rpc fail (#12290) Signed-off-by: dragondriver --- .../distributed/datacoord/client/client.go | 6 ++- .../distributed/datanode/client/client.go | 6 +-- .../distributed/indexcoord/client/client.go | 6 +-- .../distributed/indexnode/client/client.go | 6 +-- internal/distributed/proxy/client/client.go | 6 +-- .../distributed/querycoord/client/client.go | 6 +-- .../distributed/querynode/client/client.go | 6 +-- .../distributed/rootcoord/client/client.go | 6 +-- internal/util/trace/stack_trace.go | 44 +++++++++++++++++ internal/util/trace/stack_trace_test.go | 47 +++++++++++++++++++ 10 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 internal/util/trace/stack_trace.go create mode 100644 internal/util/trace/stack_trace_test.go diff --git a/internal/distributed/datacoord/client/client.go b/internal/distributed/datacoord/client/client.go index 33a6cb967..fee3ce29b 100644 --- a/internal/distributed/datacoord/client/client.go +++ b/internal/distributed/datacoord/client/client.go @@ -202,7 +202,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("DataCoord Client grpc error", zap.Error(err)) @@ -210,6 +210,10 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) c.resetConnection() ret, err = caller() + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) + } + return ret, err } diff --git a/internal/distributed/datanode/client/client.go b/internal/distributed/datanode/client/client.go index 3d3a261dd..438f217fd 100644 --- a/internal/distributed/datanode/client/client.go +++ b/internal/distributed/datanode/client/client.go @@ -183,7 +183,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("DataNode Client grpc error", zap.Error(err)) @@ -191,8 +191,8 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/indexcoord/client/client.go b/internal/distributed/indexcoord/client/client.go index 975e437ba..d1d6111e3 100644 --- a/internal/distributed/indexcoord/client/client.go +++ b/internal/distributed/indexcoord/client/client.go @@ -189,7 +189,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("IndexCoord Client grpc error", zap.Error(err)) @@ -197,8 +197,8 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/indexnode/client/client.go b/internal/distributed/indexnode/client/client.go index 521724b25..5438e8018 100644 --- a/internal/distributed/indexnode/client/client.go +++ b/internal/distributed/indexnode/client/client.go @@ -179,15 +179,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("IndexNode Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/proxy/client/client.go b/internal/distributed/proxy/client/client.go index b30a45f88..d06e6ae3d 100644 --- a/internal/distributed/proxy/client/client.go +++ b/internal/distributed/proxy/client/client.go @@ -180,15 +180,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("Proxy Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/querycoord/client/client.go b/internal/distributed/querycoord/client/client.go index 287e4819d..c0635d7cb 100644 --- a/internal/distributed/querycoord/client/client.go +++ b/internal/distributed/querycoord/client/client.go @@ -204,15 +204,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("QueryCoord Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/querynode/client/client.go b/internal/distributed/querynode/client/client.go index 93e6a6e7f..59df81c99 100644 --- a/internal/distributed/querynode/client/client.go +++ b/internal/distributed/querynode/client/client.go @@ -171,15 +171,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("QueryNode Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/rootcoord/client/client.go b/internal/distributed/rootcoord/client/client.go index ed1873882..a93c0e9b2 100644 --- a/internal/distributed/rootcoord/client/client.go +++ b/internal/distributed/rootcoord/client/client.go @@ -232,15 +232,15 @@ func (c *GrpcClient) recall(caller func() (interface{}, error)) (interface{}, er return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("RootCoord Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/util/trace/stack_trace.go b/internal/util/trace/stack_trace.go new file mode 100644 index 000000000..51d5eca0c --- /dev/null +++ b/internal/util/trace/stack_trace.go @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package trace + +import ( + "fmt" + "runtime" +) + +const numFuncsInStack = 10 + +// StackTraceMsg returns the stack information, which numFuncs means how many functions do you want to show in the stack +// information. +func StackTraceMsg(numFuncs uint) string { + pc := make([]uintptr, numFuncs) + n := runtime.Callers(0, pc) + frames := runtime.CallersFrames(pc[:n]) + + ret := "" + + for { + frame, more := frames.Next() + ret += fmt.Sprintf("%s:%d %s\n", frame.File, frame.Line, frame.Function) + if !more { + break + } + } + + return ret +} + +// StackTrace returns the stack trace information. +func StackTrace() string { + return StackTraceMsg(numFuncsInStack) +} diff --git a/internal/util/trace/stack_trace_test.go b/internal/util/trace/stack_trace_test.go new file mode 100644 index 000000000..2d6dd1f70 --- /dev/null +++ b/internal/util/trace/stack_trace_test.go @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package trace + +import ( + "fmt" + "testing" +) + +func TestStackTraceMsg(t *testing.T) { + fmt.Println(StackTraceMsg(1)) + fmt.Println(StackTraceMsg(5)) + fmt.Println(StackTraceMsg(10)) + + func() { + fmt.Println(StackTraceMsg(10)) + }() + + func() { + func() { + fmt.Println(StackTraceMsg(10)) + }() + }() +} + +func TestStackTrace(t *testing.T) { + fmt.Println(StackTrace()) + + func() { + fmt.Println(StackTrace()) + }() + + func() { + func() { + fmt.Println(StackTrace()) + }() + }() +} -- GitLab