diff --git a/.gitignore b/.gitignore
index 1bfb186f8911ab42d593746bc4a1de5feba997ee..fb7430ac853ba71401872dbfe51aad0c1f825b88 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 data
 model
+__pycache__
 .DS_Store
 .idea
-.pyc
\ No newline at end of file
+*.pyc
diff --git a/src/__pycache__/__init__.cpython-38.pyc b/src/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 4539aeb7f04ac5f61f8ab34db9f0cc4a587076b9..0000000000000000000000000000000000000000
Binary files a/src/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/src/__pycache__/model.cpython-38.pyc b/src/__pycache__/model.cpython-38.pyc
deleted file mode 100644
index 4e0528786137ec673289dabd5b468d5151d43db3..0000000000000000000000000000000000000000
Binary files a/src/__pycache__/model.cpython-38.pyc and /dev/null differ
diff --git a/src/__pycache__/model_run.cpython-38.pyc b/src/__pycache__/model_run.cpython-38.pyc
deleted file mode 100644
index 6f6614687dfff671f5abc0d2d66abb43358e4209..0000000000000000000000000000000000000000
Binary files a/src/__pycache__/model_run.cpython-38.pyc and /dev/null differ
diff --git a/src/__pycache__/utils.cpython-38.pyc b/src/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index 6c5da902d571c5208340c8cc424eb0703b89e46c..0000000000000000000000000000000000000000
Binary files a/src/__pycache__/utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 881f0098424f1fb7a13a8273dc2b8ac2626c5d44..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index 528ebc3c9c73d9d918b4d38b9c8c7499b50fd8ab..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/__pycache__/git_version_info.cpython-38.pyc b/tools/deepspeed/__pycache__/git_version_info.cpython-38.pyc
deleted file mode 100644
index 44c742f134cc9f784a353c5fbc811de21a3716a2..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/__pycache__/git_version_info.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/elasticity/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/elasticity/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 7f9074968fbe35a6e6b40449eb5486940fc4cdd7..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/elasticity/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/elasticity/__pycache__/config.cpython-38.pyc b/tools/deepspeed/elasticity/__pycache__/config.cpython-38.pyc
deleted file mode 100644
index 660eca7ca070b9ad44372f38985781d37da496aa..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/elasticity/__pycache__/config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/elasticity/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/elasticity/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index f99309c580dbcd18e4d02949cd064817c1ce71a7..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/elasticity/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/elasticity/__pycache__/elasticity.cpython-38.pyc b/tools/deepspeed/elasticity/__pycache__/elasticity.cpython-38.pyc
deleted file mode 100644
index b35fc8580d8591a4e973265acac1fd69c829a1ee..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/elasticity/__pycache__/elasticity.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/launcher/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/launcher/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 98ac337c40a224fb49067a57ec8f84f540daa451..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/launcher/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/launcher/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/launcher/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index 8db4258a97cd69ac94c2c9fb6229f567fe445fbe..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/launcher/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/launcher/__pycache__/multinode_runner.cpython-38.pyc b/tools/deepspeed/launcher/__pycache__/multinode_runner.cpython-38.pyc
deleted file mode 100644
index 85165f74cae5a7914d648f544c11f2f154a9560f..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/launcher/__pycache__/multinode_runner.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/launcher/__pycache__/runner.cpython-38.pyc b/tools/deepspeed/launcher/__pycache__/runner.cpython-38.pyc
deleted file mode 100644
index bf1cb1ae9286c46963838f33ce75b580751b5e4a..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/launcher/__pycache__/runner.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 6a987338d8d752fa4310407e229d87ad7c45a5f9..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/__pycache__/module_inject.cpython-38.pyc b/tools/deepspeed/ops/__pycache__/module_inject.cpython-38.pyc
deleted file mode 100644
index dd1e069f126d153a040a5013882b75ba642fe95a..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/__pycache__/module_inject.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/adam/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/adam/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index e54fabbb0f901d71ee5ca2818ccb94771f229f18..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/adam/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/adam/__pycache__/cpu_adam.cpython-38.pyc b/tools/deepspeed/ops/adam/__pycache__/cpu_adam.cpython-38.pyc
deleted file mode 100644
index dac6aed23ba9e744886b10839fdcada7769f01a8..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/adam/__pycache__/cpu_adam.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/adam/__pycache__/fused_adam.cpython-38.pyc b/tools/deepspeed/ops/adam/__pycache__/fused_adam.cpython-38.pyc
deleted file mode 100644
index 34a02e6c5010d6b167a5b1499683a0f4043f80d8..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/adam/__pycache__/fused_adam.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/adam/__pycache__/multi_tensor_apply.cpython-38.pyc b/tools/deepspeed/ops/adam/__pycache__/multi_tensor_apply.cpython-38.pyc
deleted file mode 100644
index a48d8c60d9323f81799280224bd4b9a28224385c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/adam/__pycache__/multi_tensor_apply.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/aio/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/aio/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index e7c963e6309f84b6d32bf912981b18a957a2d09c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/aio/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/lamb/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/lamb/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 10f2fbfe2547fcaad7462c9b12f44f32b9cbc24f..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/lamb/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/lamb/__pycache__/fused_lamb.cpython-38.pyc b/tools/deepspeed/ops/lamb/__pycache__/fused_lamb.cpython-38.pyc
deleted file mode 100644
index a1d07c29a5e512a6de01a720c8f99efb4a248f5a..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/lamb/__pycache__/fused_lamb.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index d3b60770c534dd432a939d53ee76122482023966..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/bert_sparse_self_attention.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/bert_sparse_self_attention.cpython-38.pyc
deleted file mode 100644
index 585938fffa6271cec0441065a2c31f6611e72d0f..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/bert_sparse_self_attention.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/matmul.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/matmul.cpython-38.pyc
deleted file mode 100644
index 0e2639a6071e078249bbf728ecb5e8c801d6e084..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/matmul.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/softmax.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/softmax.cpython-38.pyc
deleted file mode 100644
index 1ab8b03e0157532890f74a646337c736f73830d8..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/softmax.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/sparse_attention_utils.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/sparse_attention_utils.cpython-38.pyc
deleted file mode 100644
index f567965ee664f4872505ff25fb5a07f8d56dfa4e..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/sparse_attention_utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/sparse_self_attention.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/sparse_self_attention.cpython-38.pyc
deleted file mode 100644
index 0d8e9ac37870d18cddf51bcc343f53ea18a93f04..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/sparse_self_attention.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/__pycache__/sparsity_config.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/__pycache__/sparsity_config.cpython-38.pyc
deleted file mode 100644
index 29c0f1a74030111257c4b7358bfb70b0fc89f22c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/__pycache__/sparsity_config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/sparse_attention/trsrc/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/sparse_attention/trsrc/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 3bcc079917080ca972ab8d6afebbfd436ce0363c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/sparse_attention/trsrc/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/transformer/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/ops/transformer/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 28dcf4f7afb980f084a2d474da21d1d08017d211..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/transformer/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/ops/transformer/__pycache__/transformer.cpython-38.pyc b/tools/deepspeed/ops/transformer/__pycache__/transformer.cpython-38.pyc
deleted file mode 100644
index 15a56694198c99eb417c81c59292e7932781412f..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/ops/transformer/__pycache__/transformer.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/pipe/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/pipe/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 2b421eaedac09302253baf677f7320d4efaabc5e..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/pipe/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/profiling/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/profiling/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index e18b663904401dc9864d84651af32d84d56772fe..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/profiling/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/profiling/__pycache__/config.cpython-38.pyc b/tools/deepspeed/profiling/__pycache__/config.cpython-38.pyc
deleted file mode 100644
index 32604c3dfdee277acfaaa99663dbb4ba7482358c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/profiling/__pycache__/config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/profiling/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/profiling/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index 2ea92b38102f1abbfea34badcd7c5d1fb9bcc213..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/profiling/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/profiling/flops_profiler/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/profiling/flops_profiler/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 2093baa341fe9a7ab149cfb4415e07598cd7f224..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/profiling/flops_profiler/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/profiling/flops_profiler/__pycache__/profiler.cpython-38.pyc b/tools/deepspeed/profiling/flops_profiler/__pycache__/profiler.cpython-38.pyc
deleted file mode 100644
index 39552978a07bb72c6851173e5dea49d6241a4f4e..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/profiling/flops_profiler/__pycache__/profiler.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 04fda32178b82290cdc1fc8c4975c2f290f5be19..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/config.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/config.cpython-38.pyc
deleted file mode 100644
index 24d1c64f665f8215f0d885251f0ab0a49a043555..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/config_utils.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/config_utils.cpython-38.pyc
deleted file mode 100644
index 721bc409bd126f37d50b2505a8b382eb5b68a865..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/config_utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index 424f8031e9cd0560ac4c4a340c4cfc772887086c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/csr_tensor.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/csr_tensor.cpython-38.pyc
deleted file mode 100644
index 5d9a7afe0fce6eb83d360c33fa1ab595d0eaa4f1..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/csr_tensor.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/dataloader.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/dataloader.cpython-38.pyc
deleted file mode 100644
index e40728c2ea337fc4390440284ba665af4d4be289..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/dataloader.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/engine.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/engine.cpython-38.pyc
deleted file mode 100644
index 512b7fb2dc867a7a6e4f8149280c18d1a061bcdd..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/engine.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/lr_schedules.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/lr_schedules.cpython-38.pyc
deleted file mode 100644
index e9bf02a8b3723d86a21a8126128a02ed00536612..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/lr_schedules.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/progressive_layer_drop.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/progressive_layer_drop.cpython-38.pyc
deleted file mode 100644
index 03250a57e6378889e40353cea5318b2090aaca05..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/progressive_layer_drop.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/__pycache__/utils.cpython-38.pyc b/tools/deepspeed/runtime/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index a74f0fe29b2800f9a44ecd87ad919a6ae35b5967..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/__pycache__/utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/activation_checkpointing/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/runtime/activation_checkpointing/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 14c7478667a350e943e2c4c7f3ac315007574804..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/activation_checkpointing/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/activation_checkpointing/__pycache__/checkpointing.cpython-38.pyc b/tools/deepspeed/runtime/activation_checkpointing/__pycache__/checkpointing.cpython-38.pyc
deleted file mode 100644
index 78c5f0170c5a5f67546dd97bf3b508ac87b1ec01..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/activation_checkpointing/__pycache__/checkpointing.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/activation_checkpointing/__pycache__/config.cpython-38.pyc b/tools/deepspeed/runtime/activation_checkpointing/__pycache__/config.cpython-38.pyc
deleted file mode 100644
index f8f5b8c3bc28dad4c74c300d4448e4d60fe03718..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/activation_checkpointing/__pycache__/config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/fp16/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/runtime/fp16/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index bdc07fa60fb4bf6c0990f40593bd02631caa66a7..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/fp16/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/fp16/__pycache__/fused_optimizer.cpython-38.pyc b/tools/deepspeed/runtime/fp16/__pycache__/fused_optimizer.cpython-38.pyc
deleted file mode 100644
index 420d0bf3129c15a64d989f77ded614e493701dc4..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/fp16/__pycache__/fused_optimizer.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/fp16/__pycache__/loss_scaler.cpython-38.pyc b/tools/deepspeed/runtime/fp16/__pycache__/loss_scaler.cpython-38.pyc
deleted file mode 100644
index ae2111d1f72402016805a7f24010c2477132b236..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/fp16/__pycache__/loss_scaler.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/fp16/__pycache__/unfused_optimizer.cpython-38.pyc b/tools/deepspeed/runtime/fp16/__pycache__/unfused_optimizer.cpython-38.pyc
deleted file mode 100644
index 436f4346bce1a17b7d52a2be0977843646f5bfd0..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/fp16/__pycache__/unfused_optimizer.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/pipe/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/runtime/pipe/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index d82c86982448bbbf6ee4e961eb6ef786ff97ea2d..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/pipe/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/pipe/__pycache__/engine.cpython-38.pyc b/tools/deepspeed/runtime/pipe/__pycache__/engine.cpython-38.pyc
deleted file mode 100644
index a2e388d586b334e0be1cbb9abb44cdb3c0dba664..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/pipe/__pycache__/engine.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/pipe/__pycache__/module.cpython-38.pyc b/tools/deepspeed/runtime/pipe/__pycache__/module.cpython-38.pyc
deleted file mode 100644
index f92a1f81c267845da69dd17854443074c2bd6683..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/pipe/__pycache__/module.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/pipe/__pycache__/p2p.cpython-38.pyc b/tools/deepspeed/runtime/pipe/__pycache__/p2p.cpython-38.pyc
deleted file mode 100644
index f5ad6cc484e2b4acc2197bc2ee06e7581ee6dfcf..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/pipe/__pycache__/p2p.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/pipe/__pycache__/schedule.cpython-38.pyc b/tools/deepspeed/runtime/pipe/__pycache__/schedule.cpython-38.pyc
deleted file mode 100644
index 9da866b52286c3ff62970b8bc9a2a7821d65295b..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/pipe/__pycache__/schedule.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/pipe/__pycache__/topology.cpython-38.pyc b/tools/deepspeed/runtime/pipe/__pycache__/topology.cpython-38.pyc
deleted file mode 100644
index 631da4c5429d7ac71fcc49b86d9728e2639222ad..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/pipe/__pycache__/topology.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/swap_tensor/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/runtime/swap_tensor/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 0844a43749cf07e5f508fb8bd70d6d90507a9c43..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/swap_tensor/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/swap_tensor/__pycache__/aio_config.cpython-38.pyc b/tools/deepspeed/runtime/swap_tensor/__pycache__/aio_config.cpython-38.pyc
deleted file mode 100644
index b29051175f0825fb1cddd129c39cfb7db4c106c4..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/swap_tensor/__pycache__/aio_config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/swap_tensor/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/runtime/swap_tensor/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index 92ca381c55e813bd66196d4528d05749ea2da9c5..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/swap_tensor/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/swap_tensor/__pycache__/partitioned_param_swapper.cpython-38.pyc b/tools/deepspeed/runtime/swap_tensor/__pycache__/partitioned_param_swapper.cpython-38.pyc
deleted file mode 100644
index 277e09b82588130f9c832b396da534b27338317c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/swap_tensor/__pycache__/partitioned_param_swapper.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/swap_tensor/__pycache__/utils.cpython-38.pyc b/tools/deepspeed/runtime/swap_tensor/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index a53f644ba770bd23b36800d521543e81a1105792..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/swap_tensor/__pycache__/utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 699d63b7daf221599a1901861b594e17c95fc7b7..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/config.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/config.cpython-38.pyc
deleted file mode 100644
index 86513d2d846496b5979888db187e6a1c28f089b0..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/constants.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/constants.cpython-38.pyc
deleted file mode 100644
index 9f8c1f07d5d81b96e9da491df0a46a69a7bab732..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/linear.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/linear.cpython-38.pyc
deleted file mode 100644
index 249084585a0c49ca1c6b6b1496b32dfb3ce9a661..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/linear.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/offload_config.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/offload_config.cpython-38.pyc
deleted file mode 100644
index a88dafad86968a8afe26b55b0e2233af82f71b59..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/offload_config.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/offload_constants.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/offload_constants.cpython-38.pyc
deleted file mode 100644
index 940781d2ff863f54963eea8a823500742ba25b2f..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/offload_constants.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/partition_parameters.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/partition_parameters.cpython-38.pyc
deleted file mode 100644
index 38678f2c8312f59b31046845a5505595bc3f6d25..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/partition_parameters.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/stage1.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/stage1.cpython-38.pyc
deleted file mode 100644
index 7291735df3d0894fbdb3280df1c7ca7bae31ceae..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/stage1.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/stage2.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/stage2.cpython-38.pyc
deleted file mode 100644
index 8f9ab8e36e3370f35f327c86c777fcf161c578ed..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/stage2.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/tiling.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/tiling.cpython-38.pyc
deleted file mode 100644
index 269be921ced704f707fa3546603b9f282a96448c..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/tiling.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/runtime/zero/__pycache__/utils.cpython-38.pyc b/tools/deepspeed/runtime/zero/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index 7f9ecac3c4f430e082401e8f2b5d2c961f5741ae..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/runtime/zero/__pycache__/utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/utils/__pycache__/__init__.cpython-38.pyc b/tools/deepspeed/utils/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 8da48bab93f9b1361a841d9b7e66775ceefaaf72..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/utils/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/utils/__pycache__/distributed.cpython-38.pyc b/tools/deepspeed/utils/__pycache__/distributed.cpython-38.pyc
deleted file mode 100644
index 324be24c71635d6aada9075baa553b3368c8ef9b..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/utils/__pycache__/distributed.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/utils/__pycache__/logging.cpython-38.pyc b/tools/deepspeed/utils/__pycache__/logging.cpython-38.pyc
deleted file mode 100644
index a0b7d25603d6d9aa98859f525a89c927b97dac4f..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/utils/__pycache__/logging.cpython-38.pyc and /dev/null differ
diff --git a/tools/deepspeed/utils/__pycache__/timer.cpython-38.pyc b/tools/deepspeed/utils/__pycache__/timer.cpython-38.pyc
deleted file mode 100644
index 5f9bd0565841436d7007a242b319cc527d776250..0000000000000000000000000000000000000000
Binary files a/tools/deepspeed/utils/__pycache__/timer.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/__pycache__/__init__.cpython-38.pyc b/tools/megatron/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index f383daf7d6396e4d74b597af8a74a61d6b76ce6a..0000000000000000000000000000000000000000
Binary files a/tools/megatron/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/__pycache__/initialize.cpython-38.pyc b/tools/megatron/__pycache__/initialize.cpython-38.pyc
deleted file mode 100644
index 74c21a3932736ae47c7aacf80a2a0b4c4565b05f..0000000000000000000000000000000000000000
Binary files a/tools/megatron/__pycache__/initialize.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/__pycache__/logging.cpython-38.pyc b/tools/megatron/__pycache__/logging.cpython-38.pyc
deleted file mode 100644
index b63781ea7c9538f0c4d1645ca5ee0839d6ce0647..0000000000000000000000000000000000000000
Binary files a/tools/megatron/__pycache__/logging.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/__pycache__/utils.cpython-38.pyc b/tools/megatron/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index 371360d732700d0ba347c2f8571ef9ed0a8a6bb6..0000000000000000000000000000000000000000
Binary files a/tools/megatron/__pycache__/utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/fused_kernels/__pycache__/__init__.cpython-38.pyc b/tools/megatron/fused_kernels/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 8105f867f7d4fcad897d4f7896266e7e19e4a8bd..0000000000000000000000000000000000000000
Binary files a/tools/megatron/fused_kernels/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/__init__.cpython-38.pyc b/tools/megatron/mpu/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 9cb7d5953fcbafbf071a4c4427a34cb495b5475f..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/cross_entropy.cpython-38.pyc b/tools/megatron/mpu/__pycache__/cross_entropy.cpython-38.pyc
deleted file mode 100644
index 8dc60d9601d5d93ccd719f444dcc5d3c71e3fe4c..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/cross_entropy.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/data.cpython-38.pyc b/tools/megatron/mpu/__pycache__/data.cpython-38.pyc
deleted file mode 100644
index e5a6c828a9477d03a98e6248b99a9ab2d45cf930..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/data.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/initialize.cpython-38.pyc b/tools/megatron/mpu/__pycache__/initialize.cpython-38.pyc
deleted file mode 100644
index 71b26dec421f315f5d97f7c1740eeaf0eb49a8ef..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/initialize.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/layers.cpython-38.pyc b/tools/megatron/mpu/__pycache__/layers.cpython-38.pyc
deleted file mode 100644
index 2c4f2cecbfc71c898f38f7e291e6243b5908fe83..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/layers.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/mappings.cpython-38.pyc b/tools/megatron/mpu/__pycache__/mappings.cpython-38.pyc
deleted file mode 100644
index c39397ff5e70036a86e6760a804d274c24d86897..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/mappings.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/random.cpython-38.pyc b/tools/megatron/mpu/__pycache__/random.cpython-38.pyc
deleted file mode 100644
index d19f0eb066635d779a19a85b32f6ac7813d33a48..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/random.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/mpu/__pycache__/utils.cpython-38.pyc b/tools/megatron/mpu/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index 6cd46e320bde3f694ec6345c4acf024dbce8ed3c..0000000000000000000000000000000000000000
Binary files a/tools/megatron/mpu/__pycache__/utils.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/neox_arguments/__pycache__/__init__.cpython-38.pyc b/tools/megatron/neox_arguments/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 879fb29690b7b6ca6d3cd34342d801aedce3d768..0000000000000000000000000000000000000000
Binary files a/tools/megatron/neox_arguments/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/neox_arguments/__pycache__/arguments.cpython-38.pyc b/tools/megatron/neox_arguments/__pycache__/arguments.cpython-38.pyc
deleted file mode 100644
index ef36eedd38fbf60b5e966a339e8cc3134abec9bc..0000000000000000000000000000000000000000
Binary files a/tools/megatron/neox_arguments/__pycache__/arguments.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/neox_arguments/__pycache__/deepspeed_args.cpython-38.pyc b/tools/megatron/neox_arguments/__pycache__/deepspeed_args.cpython-38.pyc
deleted file mode 100644
index fdf6bde79552fa3455e3a2fb502ce18fe1f7863c..0000000000000000000000000000000000000000
Binary files a/tools/megatron/neox_arguments/__pycache__/deepspeed_args.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/neox_arguments/__pycache__/neox_args.cpython-38.pyc b/tools/megatron/neox_arguments/__pycache__/neox_args.cpython-38.pyc
deleted file mode 100644
index 9efec1e8da9dba76b566673bac3c42b91c00beaa..0000000000000000000000000000000000000000
Binary files a/tools/megatron/neox_arguments/__pycache__/neox_args.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/neox_arguments/__pycache__/template.cpython-38.pyc b/tools/megatron/neox_arguments/__pycache__/template.cpython-38.pyc
deleted file mode 100644
index 871ef1ae417f85cb68833d48c337c277769963bf..0000000000000000000000000000000000000000
Binary files a/tools/megatron/neox_arguments/__pycache__/template.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/tokenizer/__pycache__/__init__.cpython-38.pyc b/tools/megatron/tokenizer/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 0a292c870f6d246afac3eaf62ee3d64f0bcadb58..0000000000000000000000000000000000000000
Binary files a/tools/megatron/tokenizer/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/tokenizer/__pycache__/gpt2_tokenization.cpython-38.pyc b/tools/megatron/tokenizer/__pycache__/gpt2_tokenization.cpython-38.pyc
deleted file mode 100644
index e9c67841561e970ae27083aa75d5bedba0e372d8..0000000000000000000000000000000000000000
Binary files a/tools/megatron/tokenizer/__pycache__/gpt2_tokenization.cpython-38.pyc and /dev/null differ
diff --git a/tools/megatron/tokenizer/__pycache__/tokenizer.cpython-38.pyc b/tools/megatron/tokenizer/__pycache__/tokenizer.cpython-38.pyc
deleted file mode 100644
index 8c4b1ae3d72307939b35532c5bf22a83b7ac436b..0000000000000000000000000000000000000000
Binary files a/tools/megatron/tokenizer/__pycache__/tokenizer.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/__init__.cpython-38.pyc b/tools/op_builder/__pycache__/__init__.cpython-38.pyc
deleted file mode 100644
index 37f4df8ccb3aff013fb8cd93e171d5090bbe916c..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/__init__.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/async_io.cpython-38.pyc b/tools/op_builder/__pycache__/async_io.cpython-38.pyc
deleted file mode 100644
index 840425b5adc3014ac024172c4e59c5824779e119..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/async_io.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/builder.cpython-38.pyc b/tools/op_builder/__pycache__/builder.cpython-38.pyc
deleted file mode 100644
index ee4f6c983cfd0da6943c328eebf65dc8a8ad238f..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/builder.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/cpu_adam.cpython-38.pyc b/tools/op_builder/__pycache__/cpu_adam.cpython-38.pyc
deleted file mode 100644
index e8c7c8c8e46d53dfd7a56b06ce0c056177ca4171..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/cpu_adam.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/fused_adam.cpython-38.pyc b/tools/op_builder/__pycache__/fused_adam.cpython-38.pyc
deleted file mode 100644
index 721f558d3c655bcdeba863ed1d0a15525e99f6fa..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/fused_adam.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/fused_lamb.cpython-38.pyc b/tools/op_builder/__pycache__/fused_lamb.cpython-38.pyc
deleted file mode 100644
index ec0abfefc94a0a0af406b63790e1cd4010aa6e91..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/fused_lamb.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/sparse_attn.cpython-38.pyc b/tools/op_builder/__pycache__/sparse_attn.cpython-38.pyc
deleted file mode 100644
index 56fda97135d9e20cf3deced0b1ca9bfb20e8a77e..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/sparse_attn.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/stochastic_transformer.cpython-38.pyc b/tools/op_builder/__pycache__/stochastic_transformer.cpython-38.pyc
deleted file mode 100644
index 3f87ee3f550e1daa9168494d6058aa3e747a9b6d..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/stochastic_transformer.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/transformer.cpython-38.pyc b/tools/op_builder/__pycache__/transformer.cpython-38.pyc
deleted file mode 100644
index bbbf25c8aab784c8cd3ecc89a7fd6d836b22c32a..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/transformer.cpython-38.pyc and /dev/null differ
diff --git a/tools/op_builder/__pycache__/utils.cpython-38.pyc b/tools/op_builder/__pycache__/utils.cpython-38.pyc
deleted file mode 100644
index a2a9f0d3567e7f32a7711e7ba66441f7d5f19c8a..0000000000000000000000000000000000000000
Binary files a/tools/op_builder/__pycache__/utils.cpython-38.pyc and /dev/null differ