diff --git a/docs/apis/models.md b/docs/apis/models.md index a76dbbec33202798e7b88de75616ccf293a29c1c..f8f5cff42034ffa5d1c3ab97bb824f57560e2b90 100644 --- a/docs/apis/models.md +++ b/docs/apis/models.md @@ -184,7 +184,7 @@ paddlex.det.FasterRCNN(num_classes=81, backbone='ResNet50', with_fpn=True, aspec **参数:** > - **num_classes** (int): 包含了背景类的类别数。默认为81。 -> - **backbone** (str): FasterRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50vd', 'ResNet101', 'ResNet101vd']。默认为'ResNet50'。 +> - **backbone** (str): FasterRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50_vd', 'ResNet101', 'ResNet101_vd']。默认为'ResNet50'。 > - **with_fpn** (bool): 是否使用FPN结构。默认为True。 > - **aspect_ratios** (list): 生成anchor高宽比的可选值。默认为[0.5, 1.0, 2.0]。 > - **anchor_sizes** (list): 生成anchor大小的可选值。默认为[32, 64, 128, 256, 512]。 @@ -265,7 +265,7 @@ paddlex.det.MaskRCNN(num_classes=81, backbone='ResNet50', with_fpn=True, aspect_ **参数:** > - **num_classes** (int): 包含了背景类的类别数。默认为81。 -> - **backbone** (str): MaskRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50vd', 'ResNet101', 'ResNet101vd']。默认为'ResNet50'。 +> - **backbone** (str): MaskRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50_vd', 'ResNet101', 'ResNet101_vd']。默认为'ResNet50'。 > - **with_fpn** (bool): 是否使用FPN结构。默认为True。 > - **aspect_ratios** (list): 生成anchor高宽比的可选值。默认为[0.5, 1.0, 2.0]。 > - **anchor_sizes** (list): 生成anchor大小的可选值。默认为[32, 64, 128, 256, 512]。 diff --git a/docs/index.rst b/docs/index.rst index 3ab0bdb166bee275bd59e690027f2d0c7a0c0e78..b8c2177a2a323eb6f4dc7d75321c9ea4672db230 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,7 +22,7 @@ PaddleX是基于飞桨技术生态的深度学习全流程开发工具。具备 client_use.md FAQ.md -* PaddleX版本: v0.1.6 +* PaddleX版本: v0.1.7 * 项目官网: http://www.paddlepaddle.org.cn/paddle/paddlex * 项目GitHub: https://github.com/PaddlePaddle/PaddleX/tree/develop * 官方QQ用户群: 1045148026 diff --git a/paddlex/__init__.py b/paddlex/__init__.py index 918976a59c4fffda1bce071ffa12a03d0d7cb9b7..4c6f7e5595e38a0dcf06af6157cb221830d7f658 100644 --- a/paddlex/__init__.py +++ b/paddlex/__init__.py @@ -38,14 +38,13 @@ except: "[WARNING] pycocotools install: https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/install.md" ) -import paddlehub as hub -if hub.version.hub_version < '1.6.2': - raise Exception("[ERROR] paddlehub >= 1.6.2 is required") - +#import paddlehub as hub +#if hub.version.hub_version < '1.6.2': +# raise Exception("[ERROR] paddlehub >= 1.6.2 is required") env_info = get_environ_info() load_model = cv.models.load_model datasets = cv.datasets log_level = 2 -__version__ = '0.1.6.github' +__version__ = '0.1.7.github' diff --git a/paddlex/cv/datasets/dataset.py b/paddlex/cv/datasets/dataset.py index 09c042778147cdaef6539faaa1ef9865b149e815..c3bec8997cb0a04590f4946dc87b0eb8bd3a0c43 100644 --- a/paddlex/cv/datasets/dataset.py +++ b/paddlex/cv/datasets/dataset.py @@ -254,3 +254,11 @@ class Dataset: buffer_size=self.buffer_size, batch_size=batch_size, drop_last=drop_last) + + def set_num_samples(self, num_samples): + if num_samples > len(self.file_list): + logging.warning( + "You want set num_samples to {}, but your dataset only has {} samples, so we will keep your dataset num_samples as {}" + .format(num_samples, len(self.file_list), len(self.file_list))) + num_samples = len(self.file_list) + self.num_samples = num_samples diff --git a/paddlex/cv/models/base.py b/paddlex/cv/models/base.py index 246c1e56de585e0cfe1bb9d5c922ba99a5492f3c..beb51d7d269925bb620015dda6f94a1f8f028a92 100644 --- a/paddlex/cv/models/base.py +++ b/paddlex/cv/models/base.py @@ -199,18 +199,31 @@ class BaseAPI: self.exe.run(startup_prog) if pretrain_weights is not None: logging.info( - "Load pretrain weights from {}.".format(pretrain_weights)) + "Load pretrain weights from {}.".format(pretrain_weights), + use_color=True) paddlex.utils.utils.load_pretrain_weights( self.exe, self.train_prog, pretrain_weights, fuse_bn) # 进行裁剪 if sensitivities_file is not None: + import paddleslim from .slim.prune_config import get_sensitivities sensitivities_file = get_sensitivities(sensitivities_file, self, save_dir) from .slim.prune import get_params_ratios, prune_program + logging.info( + "Start to prune program with eval_metric_loss = {}".format( + eval_metric_loss), + use_color=True) + origin_flops = paddleslim.analysis.flops(self.test_prog) prune_params_ratios = get_params_ratios( sensitivities_file, eval_metric_loss=eval_metric_loss) prune_program(self, prune_params_ratios) + current_flops = paddleslim.analysis.flops(self.test_prog) + remaining_ratio = current_flops / origin_flops + logging.info( + "Finish prune program, before FLOPs:{}, after prune FLOPs:{}, remaining ratio:{}" + .format(origin_flops, current_flops, remaining_ratio), + use_color=True) self.status = 'Prune' def resume_checkpoint(self, path, startup_prog=None): @@ -419,7 +432,7 @@ class BaseAPI: earlystop = EarlyStop(early_stop_patience, thresh) best_accuracy_key = "" best_accuracy = -1.0 - best_model_epoch = 1 + best_model_epoch = -1 for i in range(start_epoch, num_epochs): records = list() step_start_time = time.time() @@ -492,7 +505,7 @@ class BaseAPI: current_save_dir = osp.join(save_dir, "epoch_{}".format(i + 1)) if not osp.isdir(current_save_dir): os.makedirs(current_save_dir) - if eval_dataset is not None: + if eval_dataset is not None and eval_dataset.num_samples > 0: self.eval_metrics, self.eval_details = self.evaluate( eval_dataset=eval_dataset, batch_size=eval_batch_size, @@ -524,10 +537,11 @@ class BaseAPI: self.save_model(save_dir=current_save_dir) time_eval_one_epoch = time.time() - eval_epoch_start_time eval_epoch_start_time = time.time() - logging.info( - 'Current evaluated best model in eval_dataset is epoch_{}, {}={}' - .format(best_model_epoch, best_accuracy_key, - best_accuracy)) + if best_model_epoch > 0: + logging.info( + 'Current evaluated best model in eval_dataset is epoch_{}, {}={}' + .format(best_model_epoch, best_accuracy_key, + best_accuracy)) if eval_dataset is not None and early_stop: if earlystop(current_accuracy): break diff --git a/paddlex/cv/models/slim/prune_config.py b/paddlex/cv/models/slim/prune_config.py index 7eab6b7defce11f874e95b910f2287d6a17faec7..34675a53b8bda539f3aa5cfc0adf374f449df303 100644 --- a/paddlex/cv/models/slim/prune_config.py +++ b/paddlex/cv/models/slim/prune_config.py @@ -15,7 +15,7 @@ import numpy as np import os.path as osp import paddle.fluid as fluid -import paddlehub as hub +#import paddlehub as hub import paddlex sensitivities_data = { @@ -105,22 +105,26 @@ def get_sensitivities(flag, model, save_dir): model_type) url = sensitivities_data[model_type] fname = osp.split(url)[-1] - try: - hub.download(fname, save_path=save_dir) - except Exception as e: - if isinstance(e, hub.ResourceNotFoundError): - raise Exception( - "Resource for model {}(key='{}') not found".format( - model_type, fname)) - elif isinstance(e, hub.ServerConnectionError): - raise Exception( - "Cannot get reource for model {}(key='{}'), please check your internet connecgtion" - .format(model_type, fname)) - else: - raise Exception( - "Unexpected error, please make sure paddlehub >= 1.6.2 {}". - format(str(e))) + paddlex.utils.download(url, path=save_dir) return osp.join(save_dir, fname) + + +# try: +# hub.download(fname, save_path=save_dir) +# except Exception as e: +# if isinstance(e, hub.ResourceNotFoundError): +# raise Exception( +# "Resource for model {}(key='{}') not found".format( +# model_type, fname)) +# elif isinstance(e, hub.ServerConnectionError): +# raise Exception( +# "Cannot get reource for model {}(key='{}'), please check your internet connecgtion" +# .format(model_type, fname)) +# else: +# raise Exception( +# "Unexpected error, please make sure paddlehub >= 1.6.2 {}". +# format(str(e))) +# return osp.join(save_dir, fname) else: raise Exception( "sensitivities need to be defined as directory path or `DEFAULT`(download sensitivities automatically)." diff --git a/paddlex/cv/models/utils/pretrain_weights.py b/paddlex/cv/models/utils/pretrain_weights.py index fb4d72ea007912093ac97c137b26fa0d592c789f..3f41838b7d3e1529558ced1db23e84292bdd5270 100644 --- a/paddlex/cv/models/utils/pretrain_weights.py +++ b/paddlex/cv/models/utils/pretrain_weights.py @@ -1,5 +1,5 @@ import paddlex -import paddlehub as hub +#import paddlehub as hub import os import os.path as osp @@ -85,40 +85,53 @@ def get_pretrain_weights(flag, model_type, backbone, save_dir): backbone = 'DetResNet50' assert backbone in image_pretrain, "There is not ImageNet pretrain weights for {}, you may try COCO.".format( backbone) - try: - hub.download(backbone, save_path=new_save_dir) - except Exception as e: - if isinstance(e, hub.ResourceNotFoundError): - raise Exception( - "Resource for backbone {} not found".format(backbone)) - elif isinstance(e, hub.ServerConnectionError): - raise Exception( - "Cannot get reource for backbone {}, please check your internet connecgtion" - .format(backbone)) - else: - raise Exception( - "Unexpected error, please make sure paddlehub >= 1.6.2") - return osp.join(new_save_dir, backbone) + url = image_pretrain[backbone] + fname = osp.split(url)[-1].split('.')[0] + paddlex.utils.download_and_decompress(url, path=new_save_dir) + return osp.join(new_save_dir, fname) +# try: +# hub.download(backbone, save_path=new_save_dir) +# except Exception as e: +# if isinstance(e, hub.ResourceNotFoundError): +# raise Exception( +# "Resource for backbone {} not found".format(backbone)) +# elif isinstance(e, hub.ServerConnectionError): +# raise Exception( +# "Cannot get reource for backbone {}, please check your internet connecgtion" +# .format(backbone)) +# else: +# raise Exception( +# "Unexpected error, please make sure paddlehub >= 1.6.2") +# return osp.join(new_save_dir, backbone) elif flag == 'COCO': new_save_dir = save_dir if hasattr(paddlex, 'pretrain_dir'): new_save_dir = paddlex.pretrain_dir - assert backbone in coco_pretrain, "There is not COCO pretrain weights for {}, you may try ImageNet.".format( - backbone) - try: - hub.download(backbone, save_path=new_save_dir) - except Exception as e: - if isinstance(hub.ResourceNotFoundError): - raise Exception( - "Resource for backbone {} not found".format(backbone)) - elif isinstance(hub.ServerConnectionError): - raise Exception( - "Cannot get reource for backbone {}, please check your internet connecgtion" - .format(backbone)) - else: - raise Exception( - "Unexpected error, please make sure paddlehub >= 1.6.2") - return osp.join(new_save_dir, backbone) + url = coco_pretrain[backbone] + fname = osp.split(url)[-1].split('.')[0] + paddlex.utils.download_and_decompress(url, path=new_save_dir) + return osp.join(new_save_dir, fname) + + +# new_save_dir = save_dir +# if hasattr(paddlex, 'pretrain_dir'): +# new_save_dir = paddlex.pretrain_dir +# assert backbone in coco_pretrain, "There is not COCO pretrain weights for {}, you may try ImageNet.".format( +# backbone) +# try: +# hub.download(backbone, save_path=new_save_dir) +# except Exception as e: +# if isinstance(hub.ResourceNotFoundError): +# raise Exception( +# "Resource for backbone {} not found".format(backbone)) +# elif isinstance(hub.ServerConnectionError): +# raise Exception( +# "Cannot get reource for backbone {}, please check your internet connecgtion" +# .format(backbone)) +# else: +# raise Exception( +# "Unexpected error, please make sure paddlehub >= 1.6.2") +# return osp.join(new_save_dir, backbone) else: raise Exception( "pretrain_weights need to be defined as directory path or `IMAGENET` or 'COCO' (download pretrain weights automatically)." diff --git a/setup.py b/setup.py index 99b91b86b1442a23515bfc93c326ec1498d67e27..01e778b7453daf986f6f28c0e2be2461747f8711 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ long_description = "PaddleX. A end-to-end deeplearning model development toolkit setuptools.setup( name="paddlex", - version='0.1.6', + version='0.1.7', author="paddlex", author_email="paddlex@baidu.com", description=long_description, @@ -29,9 +29,8 @@ setuptools.setup( packages=setuptools.find_packages(), setup_requires=['cython', 'numpy', 'sklearn'], install_requires=[ - "pycocotools;platform_system!='Windows'", - 'pyyaml', 'colorama', 'tqdm', 'visualdl==1.3.0', - 'paddleslim==1.0.1', 'paddlehub>=1.6.2' + "pycocotools;platform_system!='Windows'", 'pyyaml', 'colorama', 'tqdm', + 'visualdl==1.3.0', 'paddleslim==1.0.1' ], classifiers=[ "Programming Language :: Python :: 3",