automate mirror

csv_split_80%.py

+import pandas as pd
+
+csv = pd.read_csv('test.csv')
+print(len(csv))
+
+visists = csv['visits'][61:137715].to_numpy(dtype=float)
+
+sum = 0
+for index, visit in enumerate(visists):
+    sum += visit
+    if sum > visists.sum() * 0.85:
+        print(index)
+        break
+
+csv_split = csv[:index+1]
+csv_split.to_csv('csv_split.csv', columns=['project path', 'group_name', 'project_name', 'visits'], index=False)
+print(len(csv_split))
+
+
+# visists = csv['visits'][61:137715].to_numpy(dtype=float)
+# for index, num in enumerate(visists):
+#     if num < 60:
+#         print(index)
+#         break
+#
+# csv_split = csv[:index]
+# csv_split.to_csv('csv_split.csv', columns=['project path', 'group_name', 'project_name', 'visits'], index=False)
\ No newline at end of file

mirror_automate.py

+import os
+import yaml
+import pandas as pd
+
+
+csv = pd.read_csv('mirrors_needed.csv').head(20)
+
+
+for index, group in enumerate(csv['group_name']):
+    group_dir = '/home/mao/Downloads/mirrors-settings/projects/' + str(group)
+    if os.path.exists(group_dir) and os.path.exists(group_dir + '/imports.yml'):
+        print(True)
+        group_imports_yml = group_dir + '/imports.yml'
+
+        with open(group_imports_yml, 'r', encoding='utf-8') as stream:
+            try:
+                parse_yml = yaml.safe_load(stream)
+                # print(parse_yml)
+            except yaml.YAMLError as exec:
+                print(exec)
+
+        add_project_dict = {'name': str(csv['project_name'][index]), 'description': '', 'mirrorRelease': False, 'topic': ''}
+        print(add_project_dict)
+
+        projects = parse_yml['group']['projects']
+        projects.append(add_project_dict)
+
+        with open(group_imports_yml, 'w', encoding='utf-8') as stream:
+            try:
+                yaml.safe_dump(parse_yml, stream)
+            except yaml.YAMLError as exec:
+                print(exec)
+    else:
+        os.makedirs(group_dir, exist_ok=True)
+        group_imports_yml = group_dir + '/imports.yml'
+
+        add_project_dict = {'group': {'name': str(group), 'description': '', 'projects': [{'name': str(csv['project_name'][index]), 'description': '', 'mirrorRelease': False, 'topic': ''}, ]}}
+
+        with open(group_imports_yml, 'w', encoding='utf-8') as stream:
+            try:
+                yaml.safe_dump(add_project_dict, stream)
+            except yaml.YAMLError as exec:
+                print(exec)
\ No newline at end of file

mirror_group_project_extract.py

+import os
+import yaml
+import pandas as pd
+
+groups = os.listdir('/home/mao/Downloads/mirrors-settings/projects')
+
+group_list = []
+project_list = []
+
+for i, group in enumerate(groups):
+    if group == 'imports.yml':
+        continue
+    for file in os.listdir('/home/mao/Downloads/mirrors-settings/projects/' + group):
+        # print(file)
+        if len(os.listdir('/home/mao/Downloads/mirrors-settings/projects/' + group)) < 1:
+            continue
+        if file != 'imports.yml':
+            continue
+        with open('/home/mao/Downloads/mirrors-settings/projects/' + str(group) + '/' + str(file), 'r', encoding='utf-8') as stream:
+            try:
+                parse_yml = yaml.safe_load(stream)
+                # print('/home/mao/Downloads/mirrors-settings/projects/' + str(group) + '/' + str(file))
+                # print(parse_yml)
+            except yaml.YAMLError as exec:
+                print(exec)
+        # print(i, group)
+        group_name = parse_yml['group']['name']
+        for name in parse_yml['group']['projects']:
+            # print(group_name, name['name'])
+            group_list.append(group_name)
+            project_list.append(name['name'])
+
+
+mirrors = pd.DataFrame.from_dict({'group_name': group_list, 'project_name': project_list})
+
+mirrors.to_csv('mirrors.csv', columns=['group_name', 'project_name'], index=False)
\ No newline at end of file

mirrors_needed.py

+import pandas as pd
+
+csv1 = pd.read_csv('mirrors.csv')
+csv2 = pd.read_csv('csv_split.csv')
+
+add_group_list = []
+add_project_list = []
+add_project_path = []
+
+for index, group_project in enumerate(zip(csv2['group_name'], csv2['project_name'])):
+    # print(group_project[0], group_project[1])
+    if group_project[0] in list(csv1['group_name']):
+        # print(group_project[0])
+        # break
+        if group_project[1] in list(csv1['project_name']):
+            pass
+        else:
+            add_group_list.append(group_project[0])
+            add_project_list.append(group_project[1])
+            add_project_path.append(csv2['project path'].loc[index])
+    else:
+        # os.makedirs(group_project[0], exist_ok=True)
+        add_group_list.append(group_project[0])
+        add_project_list.append(group_project[1])
+        add_project_path.append(csv2['project path'].loc[index])
+
+
+mirrors_needed = pd.DataFrame.from_dict({'group_name': add_group_list, 'project_name': add_project_list, 'project path': add_project_path})
+mirrors_needed.to_csv('mirrors_needed.csv', columns=['group_name', 'project_name', 'project path'], index=False)
+print(len(mirrors_needed))
\ No newline at end of file

run_githublink.sh

+#!/bin/bash
+
+python delete_not_githublink.py
+python delete_grouplink.py
+python delete_login_settings.py
+python replace_wiki.py
+python replace_issues.py
+python replace_release.py
+python replace_blob.py
+python replace_tree.py
+python replace_archive.py
+python replace_#.py
+python replace_%.py
+python replace_.git.py
+python delete_grouplink.py
+python add_group_project_name.py
+python drop_duplicates.py
\ No newline at end of file

run_mirrorproject.sh

+#!/bin/bash
+
+python mirror_group_project_extract.py
+python csv_split_80%.py
+python mirrors_needed.py
+python mirror_automate.py
\ No newline at end of file