[vector]voxceleb1 data prepare: compatible kaldi trial and data split dir (#1412)

* compatible kaldi trial and data split dir

* remove unuseless code

dataset/voxceleb/voxceleb1.py

@@ -45,15 +45,26 @@ DATA_HOME = os.path.expanduser('.')
 # you need use --no-check-certificate to connect the target download url 
 
 BASE_URL = "https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a"
-DATA_LIST = {
+
+# dev data
+DEV_LIST = {
     "vox1_dev_wav_partaa": "e395d020928bc15670b570a21695ed96",
     "vox1_dev_wav_partab": "bbfaaccefab65d82b21903e81a8a8020",
     "vox1_dev_wav_partac": "017d579a2a96a077f40042ec33e51512",
     "vox1_dev_wav_partad": "7bb1e9f70fddc7a678fa998ea8b3ba19",
-    "vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102",
 }
+DEV_TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b"
+
+# test data
+TEST_LIST = {"vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102"}
+TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d50379a3d102"
 
-TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b"
+# kaldi trial
+# this trial file is organized by kaldi according the official file,
+# which is a little different with the official trial veri_test2.txt
+KALDI_BASE_URL = "http://www.openslr.org/resources/49/"
+TRIAL_LIST = {"voxceleb1_test_v2.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7"}
+TRIAL_TARGET_DATA = "voxceleb1_test_v2.txt voxceleb1_test_v2.txt 29fc7cc1c5d59f0816dc15d6e8be60f7"
 
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
@@ -69,7 +80,6 @@ parser.add_argument(
 
 args = parser.parse_args()
 
-
 def create_manifest(data_dir, manifest_path_prefix):
     print("Creating manifest %s ..." % manifest_path_prefix)
     json_lines = []
@@ -79,7 +89,7 @@ def create_manifest(data_dir, manifest_path_prefix):
     total_num = 0
     speakers = set()
     for audio_path in glob.glob(data_path, recursive=True):
-        audio_id = "/".join(audio_path.split("/")[-3:])
+        audio_id = "-".join(audio_path.split("/")[-3:])
         utt2spk = audio_path.split("/")[-3]
         duration = soundfile.info(audio_path).duration
         text = ""
@@ -99,14 +109,17 @@ def create_manifest(data_dir, manifest_path_prefix):
         total_num += 1
         speakers.add(utt2spk)
 
+    # data_dir_name refer to dev or test
+    # voxceleb1 is given explicit in the path
+    data_dir_name = Path(data_dir).name
+    manifest_path_prefix = manifest_path_prefix + "." + data_dir_name
     with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f:
         for line in json_lines:
             f.write(line + "\n")
 
     manifest_dir = os.path.dirname(manifest_path_prefix)
-    # data_dir_name refer to voxceleb1, which is used to distingush the voxceleb2 dataset info
-    data_dir_name = Path(data_dir).name
-    meta_path = os.path.join(manifest_dir, data_dir_name) + ".meta"
+    meta_path = os.path.join(manifest_dir, "voxceleb1." +
+                             data_dir_name) + ".meta"
     with codecs.open(meta_path, 'w', encoding='utf-8') as f:
         print(f"{total_num} utts", file=f)
         print(f"{len(speakers)} speakers", file=f)
@@ -115,10 +128,8 @@ def create_manifest(data_dir, manifest_path_prefix):
         print(f"{total_text / total_sec} text/sec", file=f)
         print(f"{total_sec / total_num} sec/utt", file=f)
 
-
 def prepare_dataset(base_url, data_list, target_dir, manifest_path,
                     target_data):
-    data_dir = os.path.join(target_dir, "voxceleb1")
     if not os.path.exists(target_dir):
         os.mkdir(target_dir)
 
@@ -126,7 +137,7 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path,
     if not os.path.exists(os.path.join(target_dir, "wav")):
         # download all dataset part
         for zip_part in data_list.keys():
-            download_url = base_url + "/" + zip_part + " --no-check-certificate "
+            download_url = " --no-check-certificate " + base_url + "/" + zip_part
             download(
                 url=download_url,
                 md5sum=data_list[zip_part],
@@ -136,8 +147,8 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path,
         all_target_part, target_name, target_md5sum = target_data.split()
         target_name = os.path.join(target_dir, target_name)
         if not os.path.exists(target_name):
-            pack_part_cmd = "cat {}/{} > {}/{}".format(
-                target_dir, all_target_part, target_dir, target_name)
+            pack_part_cmd = "cat {}/{} > {}".format(target_dir, all_target_part,
+                                                    target_name)
             subprocess.call(pack_part_cmd, shell=True)
 
         # check the target zip file md5sum
@@ -147,13 +158,11 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path,
             print("Check {} md5sum successfully".format(target_name))
 
         # unzip the all zip file
-        unzip(target_name, target_dir)
-        unzip(os.path.join(target_dir, "vox1_test_wav.zip"), target_dir)
+        if target_name.endswith(".zip"):
+            unzip(target_name, target_dir)
 
     # create the manifest file
-    create_manifest(
-        data_dir=args.target_dir, manifest_path_prefix=args.manifest_prefix)
-
+    create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path)
 
 def main():
     if args.target_dir.startswith('~'):
@@ -161,13 +170,19 @@ def main():
 
     prepare_dataset(
         base_url=BASE_URL,
-        data_list=DATA_LIST,
-        target_dir=args.target_dir,
+        data_list=DEV_LIST,
+        target_dir=os.path.join(args.target_dir, "dev"),
         manifest_path=args.manifest_prefix,
-        target_data=TARGET_DATA)
+        target_data=DEV_TARGET_DATA)
 
-    print("Manifest prepare done!")
+    prepare_dataset(
+        base_url=BASE_URL,
+        data_list=TEST_LIST,
+        target_dir=os.path.join(args.target_dir, "test"),
+        manifest_path=args.manifest_prefix,
+        target_data=TEST_TARGET_DATA)
 
+    print("Manifest prepare done!")
 
 if __name__ == '__main__':
     main()