diff --git a/internal/distributed/datacoord/client/client.go b/internal/distributed/datacoord/client/client.go index 33a6cb9672a3f37cac8faf520b88e897b836ac27..fee3ce29b5dc2328f492cc5906fdd7d352e07313 100644 --- a/internal/distributed/datacoord/client/client.go +++ b/internal/distributed/datacoord/client/client.go @@ -202,7 +202,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("DataCoord Client grpc error", zap.Error(err)) @@ -210,6 +210,10 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) c.resetConnection() ret, err = caller() + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) + } + return ret, err } diff --git a/internal/distributed/datanode/client/client.go b/internal/distributed/datanode/client/client.go index 3d3a261dd0275edcfdaabd3340da37e185f4fb90..438f217fd3852ecc28aa4087dfcc9aaf2e37a2c4 100644 --- a/internal/distributed/datanode/client/client.go +++ b/internal/distributed/datanode/client/client.go @@ -183,7 +183,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("DataNode Client grpc error", zap.Error(err)) @@ -191,8 +191,8 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/indexcoord/client/client.go b/internal/distributed/indexcoord/client/client.go index 975e437bad1e879d94c3d08a237c39fef96a9b44..d1d6111e304d45401d78a194be4aa7196334d304 100644 --- a/internal/distributed/indexcoord/client/client.go +++ b/internal/distributed/indexcoord/client/client.go @@ -189,7 +189,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("IndexCoord Client grpc error", zap.Error(err)) @@ -197,8 +197,8 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/indexnode/client/client.go b/internal/distributed/indexnode/client/client.go index 521724b2585d9dee73b78148787abaf57a6f38af..5438e8018ac7630195549ff5704b70507c0f54ed 100644 --- a/internal/distributed/indexnode/client/client.go +++ b/internal/distributed/indexnode/client/client.go @@ -179,15 +179,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("IndexNode Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/proxy/client/client.go b/internal/distributed/proxy/client/client.go index b30a45f8839e29b832239d7ed0c7bc10ead9ba8c..d06e6ae3debcebce288952ba7f0643b23444e2ee 100644 --- a/internal/distributed/proxy/client/client.go +++ b/internal/distributed/proxy/client/client.go @@ -180,15 +180,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("Proxy Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/querycoord/client/client.go b/internal/distributed/querycoord/client/client.go index 287e4819de5f9f0646787c956c3b940fb82d8540..c0635d7cbeff0a70bb487876e59777c22d9adf0d 100644 --- a/internal/distributed/querycoord/client/client.go +++ b/internal/distributed/querycoord/client/client.go @@ -204,15 +204,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("QueryCoord Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/querynode/client/client.go b/internal/distributed/querynode/client/client.go index 93e6a6e7f1368e3e4d9526240405f208ee344caa..59df81c99830a3b584c58c438e60dcddffe4f8ef 100644 --- a/internal/distributed/querynode/client/client.go +++ b/internal/distributed/querynode/client/client.go @@ -171,15 +171,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error) return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("QueryNode Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/distributed/rootcoord/client/client.go b/internal/distributed/rootcoord/client/client.go index ed187388278a88d7c4d51d24219902e69af77a06..a93c0e9b2f1b560ba1e9535ca1df40535c34b8a7 100644 --- a/internal/distributed/rootcoord/client/client.go +++ b/internal/distributed/rootcoord/client/client.go @@ -232,15 +232,15 @@ func (c *GrpcClient) recall(caller func() (interface{}, error)) (interface{}, er return ret, nil } if err == context.Canceled || err == context.DeadlineExceeded { - return nil, err + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } log.Debug("RootCoord Client grpc error", zap.Error(err)) c.resetConnection() ret, err = caller() - if err == nil { - return ret, nil + if err != nil { + return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace()) } return ret, err } diff --git a/internal/util/trace/stack_trace.go b/internal/util/trace/stack_trace.go new file mode 100644 index 0000000000000000000000000000000000000000..51d5eca0c67ba13867dd267a444a425ded6a9527 --- /dev/null +++ b/internal/util/trace/stack_trace.go @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package trace + +import ( + "fmt" + "runtime" +) + +const numFuncsInStack = 10 + +// StackTraceMsg returns the stack information, which numFuncs means how many functions do you want to show in the stack +// information. +func StackTraceMsg(numFuncs uint) string { + pc := make([]uintptr, numFuncs) + n := runtime.Callers(0, pc) + frames := runtime.CallersFrames(pc[:n]) + + ret := "" + + for { + frame, more := frames.Next() + ret += fmt.Sprintf("%s:%d %s\n", frame.File, frame.Line, frame.Function) + if !more { + break + } + } + + return ret +} + +// StackTrace returns the stack trace information. +func StackTrace() string { + return StackTraceMsg(numFuncsInStack) +} diff --git a/internal/util/trace/stack_trace_test.go b/internal/util/trace/stack_trace_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2d6dd1f70921541603bb2d1c4f15287fdf690c79 --- /dev/null +++ b/internal/util/trace/stack_trace_test.go @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package trace + +import ( + "fmt" + "testing" +) + +func TestStackTraceMsg(t *testing.T) { + fmt.Println(StackTraceMsg(1)) + fmt.Println(StackTraceMsg(5)) + fmt.Println(StackTraceMsg(10)) + + func() { + fmt.Println(StackTraceMsg(10)) + }() + + func() { + func() { + fmt.Println(StackTraceMsg(10)) + }() + }() +} + +func TestStackTrace(t *testing.T) { + fmt.Println(StackTrace()) + + func() { + fmt.Println(StackTrace()) + }() + + func() { + func() { + fmt.Println(StackTrace()) + }() + }() +}