From 0f1e7e3d52196f6c9021284d900da23b2ec6b1ce Mon Sep 17 00:00:00 2001
From: lidanqing <danqing.li@intel.com>
Date: Thu, 11 Mar 2021 08:30:40 +0100
Subject: [PATCH] [Bug fix] Different machine generate different binary file,
 remove md5 check (#31482)

* Different machine generate different binary file, remove md5 check

* remove unnecessary functions
---
 .../api/full_ILSVRC2012_val_preprocess.py     | 27 +------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py b/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py
index f25ce8be9ee..e911c942087 100644
--- a/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py
+++ b/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py
@@ -34,10 +34,8 @@ SIZE_FLOAT32 = 4
 SIZE_INT64 = 8
 FULL_SIZE_BYTES = 30106000008
 FULL_IMAGES = 50000
-TARGET_HASH = '0be07c2c23296b97dad83c626682c66a'
 FOLDER_NAME = "ILSVRC2012/"
 VALLIST_TAR_NAME = "ILSVRC2012/val_list.txt"
-CHUNK_SIZE = 8192
 
 img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
 img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
@@ -108,28 +106,6 @@ def print_processbar(done_percentage):
     sys.stdout.flush()
 
 
-def check_integrity(filename, target_hash):
-    print('\nThe binary file exists. Checking file integrity...\n')
-    md = hashlib.md5()
-    count = 0
-    onepart = FULL_SIZE_BYTES // CHUNK_SIZE // 100
-    with open(filename, 'rb') as ifs:
-        while True:
-            buf = ifs.read(CHUNK_SIZE)
-            if count % onepart == 0:
-                done = count // onepart
-                print_processbar(done)
-            count = count + 1
-            if not buf:
-                break
-            md.update(buf)
-    hash1 = md.hexdigest()
-    if hash1 == target_hash:
-        return True
-    else:
-        return False
-
-
 def convert_Imagenet_tar2bin(tar_file, output_file):
     print('Converting 50000 images to binary file ...\n')
     tar = tarfile.open(name=tar_file, mode='r:gz')
@@ -188,8 +164,7 @@ def run_convert():
     try_limit = 3
 
     while not (os.path.exists(output_file) and
-               os.path.getsize(output_file) == FULL_SIZE_BYTES and
-               check_integrity(output_file, TARGET_HASH)):
+               os.path.getsize(output_file) == FULL_SIZE_BYTES):
         if os.path.exists(output_file):
             sys.stderr.write(
                 "\n\nThe existing binary file is broken. Start to generate new one...\n\n".
-- 
GitLab