From d755b9d616bac5d830e6025dc7d0ca10d45d4ba8 Mon Sep 17 00:00:00 2001 From: Danny Semiat Date: Sat, 27 May 2023 03:17:26 +0300 Subject: [PATCH] Align InferenceEngine to store ms in _model_times (#3501) * Align InferenceEngine to store ms in _model_times When using cuda_events, the measured model time is stored in ms. When not using cuda_events, the measured model time was stored in seconds. This commit fixes the units and aligns them to store ms, the same as elapsed() function. This was observed when running the following pytest: unit/inference/test_model_profiling.py::TestModelProfiling::test[False-True-roberta-base-fill-mask] Returned values were: count=0 e2e_t=895.174312 model_t=0.8529715538024902 count=1 e2e_t=7.500252 model_t=0.0041310787200927734 count=2 e2e_t=3.887346 model_t=0.0018568038940429688 count=3 e2e_t=3.577845 model_t=0.0016334056854248047 count=4 e2e_t=3.43976 model_t=0.0016703605651855469 count=5 e2e_t=3.310903 model_t=0.0016107559204101562 count=6 e2e_t=3.299556 model_t=0.001603841781616211 count=7 e2e_t=3.605722 model_t=0.0015969276428222656 count=8 e2e_t=3.273741 model_t=0.0015516281127929688 count=9 e2e_t=3.46306 model_t=0.0016617774963378906 The units difference is observed here, when model_t is in ther order of 10e-3 comparing to e2e_t * Update engine.py --------- Co-authored-by: Michael Wyatt --- deepspeed/inference/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepspeed/inference/engine.py b/deepspeed/inference/engine.py index c029aaa7..d2f9bf9c 100755 --- a/deepspeed/inference/engine.py +++ b/deepspeed/inference/engine.py @@ -253,7 +253,7 @@ class InferenceEngine(Module): else: get_accelerator().synchronize() self._end = time.time() - elapsed_time = self._end - self._start + elapsed_time = (self._end - self._start) * 1e3 # convert seconds to ms self._model_times.append(elapsed_time) def _create_model_parallel_group(self, config): @@ -612,7 +612,7 @@ class InferenceEngine(Module): if self.model_profile_enabled and self._config.enable_cuda_graph: get_accelerator().synchronize() - duration = time.time() - start + duration = (time.time() - start) * 1e3 # convert seconds to ms self._model_times.append(duration) return outputs -- GitLab