diff --git a/demo/image-classification/run_classifier.sh b/demo/image-classification/run_classifier.sh index d91e67ab1b8e4fd01144734c5def5fe7eee3e7e4..bcb6dc327e501ff2259289c65fdb0a87e887144d 100644 --- a/demo/image-classification/run_classifier.sh +++ b/demo/image-classification/run_classifier.sh @@ -1,2 +1,4 @@ export FLAGS_eager_delete_tensor_gb=0.0 +export CUDA_VISIBLE_DEVICES=0 + python -u img_classifier.py $@ diff --git a/demo/image-classification/run_predict.sh b/demo/image-classification/run_predict.sh index 149c89cfc328360ff351d5547d8974a682af58f1..1c6890aa337dcef215f2589ff421c1b3b6ba0bdb 100644 --- a/demo/image-classification/run_predict.sh +++ b/demo/image-classification/run_predict.sh @@ -1,2 +1,4 @@ export FLAGS_eager_delete_tensor_gb=0.0 +export CUDA_VISIBLE_DEVICES=0 + python -u predict.py $@ diff --git a/demo/qa_classification/run_classifier.sh b/demo/qa_classification/run_classifier.sh index e41b4a50699e1b2b4a39a95f92e82fed7c2a023b..6f6abc7b874d2431bcf928342c4d3f9784a8c115 100644 --- a/demo/qa_classification/run_classifier.sh +++ b/demo/qa_classification/run_classifier.sh @@ -1,7 +1,6 @@ export FLAGS_eager_delete_tensor_gb=0.0 export CUDA_VISIBLE_DEVICES=0 - CKPT_DIR="./ckpt_qa" # Recommending hyper parameters for difference task # ChnSentiCorp: batch_size=24, weight_decay=0.01, num_epoch=3, max_seq_len=128, lr=5e-5 diff --git a/demo/reading-comprehension/reading_comprehension.py b/demo/reading-comprehension/reading_comprehension.py index 93298b7f0aa69f003d949e68673a87ef1040f9e2..85bdada1d86cb0e61f08993c33ea46191b12abad 100644 --- a/demo/reading-comprehension/reading_comprehension.py +++ b/demo/reading-comprehension/reading_comprehension.py @@ -89,9 +89,7 @@ if __name__ == '__main__': # Setup runing config for PaddleHub Finetune API config = hub.RunConfig( - log_interval=10, eval_interval=300, - save_ckpt_interval=10000, use_pyreader=args.use_pyreader, use_data_parallel=args.use_data_parallel, use_cuda=args.use_gpu, diff --git a/demo/reading-comprehension/run_finetune.sh b/demo/reading-comprehension/run_finetune.sh index e0496f7d97b8982a27b9d99df7cd2037f24567c8..9d92042c1d4f645b00832befa03385f5af3b068c 100644 --- a/demo/reading-comprehension/run_finetune.sh +++ b/demo/reading-comprehension/run_finetune.sh @@ -1,4 +1,5 @@ export FLAGS_eager_delete_tensor_gb=0.0 +export CUDA_VISIBLE_DEVICES=0 # Recommending hyper parameters for difference task # squad: batch_size=8, weight_decay=0, num_epoch=3, max_seq_len=512, lr=5e-5 diff --git a/demo/reading-comprehension/run_predict.sh b/demo/reading-comprehension/run_predict.sh index 6be03eba3a27fb2e563b4aa4276540b1a6934ce5..456f7dc2050a7d9889a29b5030dd265c2f1c569e 100644 --- a/demo/reading-comprehension/run_predict.sh +++ b/demo/reading-comprehension/run_predict.sh @@ -1,4 +1,5 @@ export FLAGS_eager_delete_tensor_gb=0.0 +export CUDA_VISIBLE_DEVICES=0 CKPT_DIR="./ckpt_cmrc2018" dataset=cmrc2018 diff --git a/demo/regression/run_predict.sh b/demo/regression/run_predict.sh index 3d0c1ae007ea12803278d29b4a94e15ad757f65b..34ce6fe34c9107d6a8ec716434971a5d125ced82 100644 --- a/demo/regression/run_predict.sh +++ b/demo/regression/run_predict.sh @@ -1,5 +1,5 @@ export FLAGS_eager_delete_tensor_gb=0.0 -# export CUDA_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=0 # User can select chnsenticorp, nlpcc_dbqa, lcqmc and so on for different task DATASET="STS-B" diff --git a/demo/sequence-labeling/predict.py b/demo/sequence-labeling/predict.py index 96fea4fa533ff585084030d930ff541a23376295..8141919677d6c1cd3ccfec056f2ed187ca48d295 100644 --- a/demo/sequence-labeling/predict.py +++ b/demo/sequence-labeling/predict.py @@ -41,7 +41,7 @@ args = parser.parse_args() if __name__ == '__main__': # loading Paddlehub ERNIE pretrained model - module = hub.Module(name="ernie") + module = hub.Module(name="ernie_tiny") inputs, outputs, program = module.context(max_seq_len=args.max_seq_len) # Sentence labeling dataset reader @@ -49,7 +49,9 @@ if __name__ == '__main__': reader = hub.reader.SequenceLabelReader( dataset=dataset, vocab_path=module.get_vocab_path(), - max_seq_len=args.max_seq_len) + max_seq_len=args.max_seq_len, + sp_model_path=module.get_spm_path(), + word_dict_path=module.get_word_dict_path()) inv_label_map = {val: key for key, val in reader.label_map.items()} place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() diff --git a/demo/sequence-labeling/run_sequence_label.sh b/demo/sequence-labeling/run_sequence_label.sh index 4b4b5c3a4b2359d61588020bf3da6c6ca21a8add..ab5eef9050fbe20949c55292d30ee49c9bb4b997 100644 --- a/demo/sequence-labeling/run_sequence_label.sh +++ b/demo/sequence-labeling/run_sequence_label.sh @@ -1,4 +1,5 @@ export FLAGS_eager_delete_tensor_gb=0.0 +export CUDA_VISIBLE_DEVICES=0 CKPT_DIR="./ckpt_sequence_label" python -u sequence_label.py \ diff --git a/demo/sequence-labeling/sequence_label.py b/demo/sequence-labeling/sequence_label.py index 00b2fe8a5abf6abbeaad8336b773b30daa2a44ab..52cfb665fe8ca65e384180904a118c21ea01db43 100644 --- a/demo/sequence-labeling/sequence_label.py +++ b/demo/sequence-labeling/sequence_label.py @@ -71,9 +71,6 @@ if __name__ == '__main__': # Setup runing config for PaddleHub Finetune API config = hub.RunConfig( - log_interval=10, - eval_interval=300, - save_ckpt_interval=10000, use_data_parallel=args.use_data_parallel, use_pyreader=args.use_pyreader, use_cuda=args.use_gpu, diff --git a/demo/text-classification/predict.py b/demo/text-classification/predict.py index 5829fd64b90a8a6f2d33d3197816707bfdb57fcf..b044a81572561e258f240ee6544f31b544fbaccf 100644 --- a/demo/text-classification/predict.py +++ b/demo/text-classification/predict.py @@ -45,15 +45,35 @@ if __name__ == '__main__': # Download dataset and use ClassifyReader to read dataset if args.dataset.lower() == "chnsenticorp": dataset = hub.dataset.ChnSentiCorp() - module = hub.Module(name="ernie") + module = hub.Module(name="ernie_tiny") + metrics_choices = ["acc"] + elif args.dataset.lower() == "tnews": + dataset = hub.dataset.TNews() + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") metrics_choices = ["acc"] elif args.dataset.lower() == "nlpcc_dbqa": dataset = hub.dataset.NLPCC_DBQA() - module = hub.Module(name="ernie") + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") metrics_choices = ["acc"] elif args.dataset.lower() == "lcqmc": dataset = hub.dataset.LCQMC() - module = hub.Module(name="ernie") + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") + metrics_choices = ["acc"] + elif args.dataset.lower() == 'inews': + dataset = hub.dataset.INews() + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") + metrics_choices = ["acc"] + elif args.dataset.lower() == 'bq': + dataset = hub.dataset.BQ() + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") + metrics_choices = ["acc"] + elif args.dataset.lower() == 'thucnews': + dataset = hub.dataset.THUCNEWS() + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") + metrics_choices = ["acc"] + elif args.dataset.lower() == 'iflytek': + dataset = hub.dataset.IFLYTEK() + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") metrics_choices = ["acc"] elif args.dataset.lower() == "mrpc": dataset = hub.dataset.GLUE("MRPC") @@ -90,7 +110,7 @@ if __name__ == '__main__': metrics_choices = ["acc"] elif args.dataset.lower().startswith("xnli"): dataset = hub.dataset.XNLI(language=args.dataset.lower()[-2:]) - module = hub.Module(name="bert_multi_cased_L-12_H-768_A-12") + module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") metrics_choices = ["acc"] else: raise ValueError("%s dataset is not defined" % args.dataset) diff --git a/demo/text-classification/run_classifier.sh b/demo/text-classification/run_classifier.sh index d297cb7493d0990064f8fa79528a6fef0dcb971e..c7e5d3295872d2f9006f426178b4f183d1ff4c7a 100644 --- a/demo/text-classification/run_classifier.sh +++ b/demo/text-classification/run_classifier.sh @@ -1,4 +1,5 @@ export FLAGS_eager_delete_tensor_gb=0.0 +export CUDA_VISIBLE_DEVICES=0 # User can select chnsenticorp, nlpcc_dbqa, lcqmc and so on for different task DATASET="chnsenticorp" diff --git a/demo/text-classification/run_predict.sh b/demo/text-classification/run_predict.sh index 281b85874c9c6fce448f8be3d44ec0f8c229d7fb..f8badbb05c6ac6260caa88f0d40176d738472178 100644 --- a/demo/text-classification/run_predict.sh +++ b/demo/text-classification/run_predict.sh @@ -17,4 +17,4 @@ python -u predict.py --checkpoint_dir=$CKPT_DIR \ --max_seq_len=128 \ --use_gpu=True \ --dataset=${DATASET} \ - --batch_size=150 \ + --batch_size=32 \ diff --git a/demo/text-classification/text_classifier.py b/demo/text-classification/text_classifier.py index 178efa9d67aa9e5e26f3570a8071347ca9db9204..155d9e6d3ece4ace28f048030d700402803d2168 100644 --- a/demo/text-classification/text_classifier.py +++ b/demo/text-classification/text_classifier.py @@ -47,7 +47,7 @@ if __name__ == '__main__': elif args.dataset.lower() == "tnews": dataset = hub.dataset.TNews() module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") - metrics_choices = ["acc", "f1"] + metrics_choices = ["acc"] elif args.dataset.lower() == "nlpcc_dbqa": dataset = hub.dataset.NLPCC_DBQA() module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") @@ -59,19 +59,19 @@ if __name__ == '__main__': elif args.dataset.lower() == 'inews': dataset = hub.dataset.INews() module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") - metrics_choices = ["acc", "f1"] + metrics_choices = ["acc"] elif args.dataset.lower() == 'bq': dataset = hub.dataset.BQ() module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") - metrics_choices = ["acc", "f1"] + metrics_choices = ["acc"] elif args.dataset.lower() == 'thucnews': dataset = hub.dataset.THUCNEWS() module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") - metrics_choices = ["acc", "f1"] + metrics_choices = ["acc"] elif args.dataset.lower() == 'iflytek': dataset = hub.dataset.IFLYTEK() module = hub.Module(name="roberta_wwm_ext_chinese_L-24_H-1024_A-16") - metrics_choices = ["acc", "f1"] + metrics_choices = ["acc"] elif args.dataset.lower() == "mrpc": dataset = hub.dataset.GLUE("MRPC") module = hub.Module(name="ernie_v2_eng_base") @@ -97,7 +97,7 @@ if __name__ == '__main__': dataset = hub.dataset.GLUE("RTE") module = hub.Module(name="ernie_v2_eng_base") metrics_choices = ["acc"] - elif args.dataset.lower() == "mnli" or args.dataset.lower() == "mnli": + elif args.dataset.lower() == "mnli" or args.dataset.lower() == "mnli_m": dataset = hub.dataset.GLUE("MNLI_m") module = hub.Module(name="ernie_v2_eng_base") metrics_choices = ["acc"] diff --git a/paddlehub/reader/cv_reader.py b/paddlehub/reader/cv_reader.py index 04fe4ead9f6a91e82fe87abd52aca5441f638aa4..8196215a1c6fcea9af7e2743a17421934e435902 100644 --- a/paddlehub/reader/cv_reader.py +++ b/paddlehub/reader/cv_reader.py @@ -49,6 +49,7 @@ class ImageClassificationReader(object): self.data_augmentation = data_augmentation self.images_std = images_std self.images_mean = images_mean + self.num_examples = {'train': -1, 'dev': -1, 'test': -1} if self.images_mean is None: try: @@ -80,12 +81,15 @@ class ImageClassificationReader(object): raise ValueError("The dataset is none and it's not allowed!") if phase == "train": data = self.dataset.train_data(shuffle) + self.num_examples['train'] = len(self.get_train_examples()) elif phase == "test": shuffle = False data = self.dataset.test_data(shuffle) + self.num_examples['test'] = len(self.get_test_examples()) elif phase == "val" or phase == "dev": shuffle = False data = self.dataset.validate_data(shuffle) + self.num_examples['dev'] = len(self.get_dev_examples()) elif phase == "predict": data = data