diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ab8d75f0a580e19ead3708040ceb897a26c754a..f09022996d84488a297d5067f839e493d5c56872 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,32 @@ -- repo: https://github.com/PaddlePaddle/mirrors-yapf.git - sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37 - hooks: - - id: yapf - files: \.py$ -- repo: https://github.com/pre-commit/pre-commit-hooks - sha: a11d9314b22d8f8c7556443875b731ef05965464 - hooks: - - id: check-merge-conflict - - id: check-symlinks - - id: detect-private-key - files: (?!.*paddle)^.*$ - - id: end-of-file-fixer - files: \.(md|yml)$ - - id: trailing-whitespace - files: \.(md|yml)$ -- repo: https://github.com/Lucas-C/pre-commit-hooks - sha: v1.0.1 - hooks: - - id: forbid-crlf - files: \.(md|yml)$ - - id: remove-crlf - files: \.(md|yml)$ - - id: forbid-tabs - files: \.(md|yml)$ - - id: remove-tabs - files: \.(md|yml)$ +- repo: https://github.com/PaddlePaddle/mirrors-yapf.git + sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37 + hooks: + - id: yapf + files: \.py$ + +- repo: https://github.com/pre-commit/pre-commit-hooks + sha: a11d9314b22d8f8c7556443875b731ef05965464 + hooks: + - id: flake8 + args: ['--ignore=E265'] + - id: check-yaml + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + files: (?!.*paddle)^.*$ + - id: end-of-file-fixer + files: \.(md|yml)$ + - id: trailing-whitespace + files: \.(md|yml)$ + +- repo: https://github.com/Lucas-C/pre-commit-hooks + sha: v1.0.1 + hooks: + - id: forbid-crlf + files: \.(md|yml)$ + - id: remove-crlf + files: \.(md|yml)$ + - id: forbid-tabs + files: \.(md|yml)$ + - id: remove-tabs + files: \.(md|yml)$ diff --git a/ppcls/utils/__init__.py b/ppcls/utils/__init__.py index 3df1283ae5f3800fb596743727a09a17b6f573fc..c4e3d25584f9550550c563d3fffdff78cc17880d 100644 --- a/ppcls/utils/__init__.py +++ b/ppcls/utils/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import environment from . import model_zoo from . import misc from . import logger diff --git a/ppcls/utils/environment.py b/ppcls/utils/environment.py deleted file mode 100644 index e6061ecea5d7e58b793f68fc0bbb3e92cdc437a3..0000000000000000000000000000000000000000 --- a/ppcls/utils/environment.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import os -import paddle -import paddle.fluid as fluid -import paddle.fluid.framework as pff - -trainers_num = int(os.environ.get('PADDLE_TRAINERS_NUM', 1)) -trainer_id = int(os.environ.get("PADDLE_TRAINER_ID", 0)) - - -def place(): - gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) - return fluid.CUDAPlace(gpu_id) - - -def places(): - """ - Returns available running places, the numbers are usually - indicated by 'export CUDA_VISIBLE_DEVICES= ' - Args: - """ - - if trainers_num <= 1: - return pff.cuda_places() - else: - return place() diff --git a/tools/eval.py b/tools/eval.py index 9fae3044c3c13c41da7315588e8d60a5cf1e2750..ffae0300802818cc6a635ccc582e8ac05da2a89d 100644 --- a/tools/eval.py +++ b/tools/eval.py @@ -1,34 +1,31 @@ -#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. +# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from __future__ import absolute_import from __future__ import division from __future__ import print_function -import sys +import os import argparse -import paddle import paddle.fluid as fluid import program from ppcls.data import Reader -import ppcls.utils.environment as env from ppcls.utils.config import get_config -from ppcls.utils.save_load import init_model, save_model -from ppcls.utils import logger +from ppcls.utils.save_load import init_model from paddle.fluid.incubate.fleet.collective import fleet from paddle.fluid.incubate.fleet.base import role_maker @@ -40,7 +37,7 @@ def parse_args(): '-c', '--config', type=str, - default='configs/eval.yaml', + default='./configs/eval.yaml', help='config file path') parser.add_argument( '-o', @@ -58,7 +55,8 @@ def main(args): fleet.init(role) config = get_config(args.config, overrides=args.override, show=True) - place = env.place() + gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) + place = fluid.CUDAPlace(gpu_id) startup_prog = fluid.Program() valid_prog = fluid.Program() diff --git a/tools/train.py b/tools/train.py index ec6f62802855adcc5a52554fad81f486b65ca482..83790a87e399b954d82c4e1d0061f602118192b9 100644 --- a/tools/train.py +++ b/tools/train.py @@ -1,16 +1,16 @@ -#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. +# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from __future__ import absolute_import from __future__ import division @@ -18,18 +18,14 @@ from __future__ import print_function import argparse import os -import sys -import paddle import paddle.fluid as fluid import program from ppcls.data import Reader -import ppcls.utils.environment as env from ppcls.utils.config import get_config from ppcls.utils.save_load import init_model, save_model -from ppcls.utils import logger from paddle.fluid.incubate.fleet.collective import fleet from paddle.fluid.incubate.fleet.base import role_maker @@ -41,7 +37,7 @@ def parse_args(): '-c', '--config', type=str, - default='configs/ResNet/ResNet18_vd.yaml', + default='configs/ResNet/ResNet50.yaml', help='config file path') parser.add_argument( '-o', @@ -58,8 +54,12 @@ def main(args): fleet.init(role) config = get_config(args.config, overrides=args.override, show=True) - place = env.place() + # assign the place + gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) + place = fluid.CUDAPlace(gpu_id) + # startup_prog is used to do some parameter init work, + # and train prog is used to hold the network startup_prog = fluid.Program() train_prog = fluid.Program() @@ -70,11 +70,15 @@ def main(args): valid_prog = fluid.Program() valid_dataloader, valid_fetchs = program.build( config, valid_prog, startup_prog, is_train=False) + # clone to prune some content which is irrelevant in valid_prog valid_prog = valid_prog.clone(for_test=True) - exe = fluid.Executor(place) + # create the "Executor" with the statement of which place + exe = fluid.Executor(place=place) + # only run startup_prog once to init exe.run(startup_prog) + # load model from checkpoint or pretrained model init_model(config, train_prog, exe) train_reader = Reader(config, 'train')() @@ -87,13 +91,15 @@ def main(args): compiled_train_prog = fleet.main_program for epoch_id in range(config.epochs): + # 1. train with train dataset program.run(train_dataloader, exe, compiled_train_prog, train_fetchs, epoch_id, 'train') - + # 2. validate with validate dataset if config.validate and epoch_id % config.valid_interval == 0: program.run(valid_dataloader, exe, compiled_valid_prog, valid_fetchs, epoch_id, 'valid') + # 3. save the persistable model if epoch_id % config.save_interval == 0: model_path = os.path.join(config.model_save_dir, config.ARCHITECTURE["name"])