@@ -24,82 +23,119 @@ from paddle.fluid import core
...
@@ -24,82 +23,119 @@ from paddle.fluid import core
frompaddle.fluid.frameworkimportdygraph_only
frompaddle.fluid.frameworkimportdygraph_only
__all__=[
__all__=[
"DebugMode",
"TensorCheckerConfig",
"enable_operator_stats_collection",
"enable_operator_stats_collection",
"disable_operator_stats_collection",
"disable_operator_stats_collection",
"collect_operator_stats",
"collect_operator_stats",
"enable_tensor_checker",
"disable_tensor_checker",
]
]
classDebugMode(Enum):
classDebugMode(Enum):
"""
The DebugMode is a feature that helps to present the state of the TensorCheckerConfig. Each DebugMode has a specific meaning, which is explained below:
- DebugMode.CHECK_NAN_INF_AND_ABORT: This mode prints or saves information about Tensors that contain NaN/Inf and interrupts the program.
- DebugMode.CHECK_NAN_INF: This mode prints or saves critical information about Tensors that contain NaN/Inf but allows the program to continue running.
- DebugMode.CHECK_ALL_FOR_OVERFLOW: This mode checks the output of the FP32 operator and prints or saves information about key Tensors that exceed the FP16 representation range, such as overflow or underflow.
- DebugMode.CHECK_ALL: This mode prints or saves output Tensor key information for all operators.
raiseValueError("skipped_op_list must be list or tuple")
classTensorCheckerConfig:
classTensorCheckerConfig:
"""
"""
Collect the config for checking nan and inf in module or op tensor.
The purpose of this class is to collect the configuration for checking NaN and Inf values in the tensors of a module or operator. It takes the following arguments:
Args:
Args:
* enable: Whether to enable Tensor's value detection function. The default value is False, which means that these tools will never be used.
enable(bool): Indicating whether to enable the detection of NaN and Inf values in tensors. The default value is False, which means that these tools will not be used.
* debug_mode: Debug mode,There are 6 kinds of debug mode.
debug_mode(DebugMode, optional): A parameter that determines the type of debugging to be used. Default is DebugMode.CHECK_NAN_INF_AND_ABORT.
CHECK_NAN_INF_AND_ABORT(default): Print or save Tensor key information with NaN/Inf and interrupt the program
CHECK_NAN_INF: Print or save Tensor critical information with NaN/Inf, but continue to run
CHECK_ALL_AND_ABORT: Print or save the output Tensor key information of all operators, and interrupt the program if NaN/Inf occurs
CHECK_ALL_FOR_OVERFLOW: Check the output of the FP32 operator, print or save key Tensor information that exceeds the FP16 representation range (overflow, underflow)
CHECK_ALL: Print or save output Tensor key information for all operators
DUMP_ALL: Saves all Tensor data. This mode does not print on the terminal
* dump_dir: The collection data storage path. If it is None, it will be directly printed to the terminal
output_dir(string, optional): The path to store collected data. If this parameter is set to None, the data will be printed to the terminal. Default is None.
* checked_op_list: A list of operators you want to check
checked_op_list(list|tuple, optional): Specifies a list of operators that need to be checked during program execution, for example, checked_op_list=['elementwise_add', 'conv2d'], indicating that the output results of elementwise_add and conv2d should be checked for nan/inf during program execution. Default is None.
* skipped_op_list: A list of operators to skip checking
skipped_op_list(list|tuple, optional): Specifies a list of operators that do not need to be checked during program execution, for example, skipped_op_list=['elementwise_add', 'conv2d'], indicating that the output results of elementwise_add and conv2d should not be checked for nan/inf during program execution. None is None.
* debug_step: The iteration scope of debugging
debug_step(list|tuple, optional): A list or tuple used primarily for nan/inf checking during model training. For example, debug_step=[1,5] indicates that nan/inf checking should only be performed on model training iterations 1 to 5. Default is None.
* stack_height_limit: The maximum depth of the call stack, and supports printing the call stack at the error location. The specific scheme needs to be investigated
stack_height_limit(int, optional): An integer value specifying the maximum depth of the call stack. This feature supports printing the call stack at the error location. Currently, only enabling or disabling call stack printing is supported. If you want to print the corresponding C++ call stack when NaN is detected in GPU Kernel, set stack_height_limit to 1, otherwise set it to 0. Default is 1.
* enable_traceback_filtering: Whether to filter the traceback. The main purpose is to filter out the internal code call stack of the framework and only display the user code call stack
enable_tensor_checker(checker_config) is enables model level accuracy checking, which is used together with disables_tensor_checker() to achieve model level precision checking through the combination of these two APIs, checking the output Tensors of all operators within the specified range.
The enable_tensor_checker(checker_config) function enables model-level accuracy checking and is used in combination with disables_tensor_checker() to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
Attention:
Args:
checker_config(TensorCheckerConfig): Checker_config is to collect the configuration for checking NaN and Inf values in the tensors of a module or operator.
* If disable is called before loss. backward()_tensor_checker(), the gradient operator is not checked;
* If disable is called before optimizer.step() tensor_checker(), the optimizer and other weight update related operators will not be checked
Note:
If disable_tensor_checker() is called before backward(), the gradient operator will not be checked.
If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked.
# when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
# Traceback (most recent call last):
# File "tp.py", line 8, in <module>
# res = paddle.pow(x, y)
# File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
# return _C_ops.elementwise_pow(x, y)
paddle.amp.debugging.disable_tensor_checker()
"""
"""
ifchecker_config.check():
ifchecker_config.update_and_check_step_id():
checker_config.run()
checker_config.start_check_nan_inf()
else:
else:
checker_config.end()
checker_config.stop_check_nan_inf()
defdisable_tensor_checker():
defdisable_tensor_checker():
"""
"""
disable_tensor_checker() to disables the accuracy checking, which is used together with enables_tensor_checker(config) to achieve model level precision checking through the combination of these two APIs, checking the output Tensors of all operators within the specified range.
disable_tensor_checker() is used to disable accuracy checking, and is used together with enable_tensor_checker(config) to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
Attention:
Note:
If disable_tensor_checker() is called before backward(), the gradient operator will not be checked;
If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked.
* If disable_tensor_checker() is called before loss.backward(), the gradient operator is not checked;
Examples:
* If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked
