import os import re import sys import uuid import json import shutil from collections import Counter def get_files_path(file_dir, filetype='.txt'): """得到文件夹下的所有.txt文件的路径 Args: file_dir: 文件夹路径 filetype: 文件后缀 Returns: 所有filetype类型文件的绝对路径 """ files_path = [] for root, dirs, files in os.walk(file_dir): for file in files: if filetype is None or (os.path.splitext(file)[1] == filetype): files_path.append(os.path.join(root, file)) return files_path def load_json(path): """ load_json(path:str)->jsObject 从指定文件读取内容，解析为 json 返回 @param path: 文件路径 @return: 解析后的 json """ with open(path) as f: data = f.read() return json.loads(data) def dump_json(path, data): """ dump_json(path:str, data:obj)->None 从指定文件读取内容，解析为 json 返回 @param path: 文件路径 @param data: json 对象 @return: None """ with open(path, "w+") as df: df.write(json.dumps(data, indent=2, ensure_ascii=False)) def extract_it_knowledge(): data_dir = 'data_backup/it_knowledge' save_dir = 'data/1.dailycode初阶/4.it_knowledge' shutil.rmtree(save_dir) os.mkdir(save_dir) files = get_files_path(data_dir, '.md') language = 'json' for file in files: with open(file, 'r', encoding='utf-8') as f: data = f.read() file_name = file.split('/')[-1].split('.')[0] # print(file_name) # print(data) question_title = re.findall(r'## 标题\n(.*?)\n##', data, re.S)[0] question_title = question_title.strip() # print(question_title) question_content = re.findall(r'## 描述\n(.*?)\n##', data, re.S)[0] question_content = question_content.strip() # if question_content == []: # print(file) keywords = re.findall(r'## 关键词\n(.*?)\n##', data, re.S)[0] keywords = keywords.strip() keywords_list = keywords.split('；') keywords = ','.join(keywords_list) # print(keywords) topic_link = re.findall(r'## 链接\n(.*?)\n##', data, re.S)[0] topic_link = topic_link.strip() # print(topic_link) difficulty = '简单' choice = re.findall(r'## 选项\n(.*?)\n##', data, re.S)[0] choice_list = choice.split('\n') choice_list_res = [] for tem in choice_list: if tem == '': continue else: tem = tem.strip() choice_list_res.append(tem) # print(choice_list_res) answer = re.findall(r'## 答案\n(.*)', data, re.S)[0] answer = answer.strip() print(file) assert answer in choice_list_res question_id = file_name choice_list_remove = [] for idx, val in enumerate(choice_list_res): if val == answer: answer_idx = idx continue else: choice_list_remove.append(val) config_data = {} config_data['node_id'] = 'dailycode-' + uuid.uuid4().hex config_data['keywords'] = [] config_data['children'] =[] config_data['export'] = ["solution.json"] solution_json_data = {} solution_json_data['type'] = 'code_options' solution_json_data['author'] = 'Byeweiyang' solution_json_data['source'] = 'solution.md' solution_json_data['exercise_id'] = uuid.uuid4().hex solution_json_data['keywords'] = keywords solution_json_data['topic_link'] = topic_link solution_md_data = f"# {question_title}\n\n{question_content}\n\n## 答案\n\n```{language}\n{answer}\n```\n\n## 选项\n\n### A\n\n```{language}\n{choice_list_remove[0]}\n```\n\n### B\n\n```{language}\n{choice_list_remove[1]}\n```\n\n### C\n\n```{language}\n{choice_list_remove[2]}\n```" print(solution_md_data) print(config_data) print(solution_json_data) dir_list_ = os.listdir(save_dir) dir_list = [] for i in dir_list_: current_dst_dir_ = os.path.join(save_dir, i) if os.path.isdir(current_dst_dir_): dir_list.append(current_dst_dir_) number = len(dir_list) + 1 dst_dir = os.path.join(save_dir, str(number) + '.exercises') print(dst_dir) solution_json_path = os.path.join(dst_dir, 'solution.json') solution_md_path = os.path.join(dst_dir, 'solution.md') config_path = os.path.join(dst_dir, 'config.json') if not os.path.exists(dst_dir): os.makedirs(dst_dir) dump_json(solution_json_path, solution_json_data) dump_json(config_path, config_data) with open(solution_md_path, 'w', encoding='utf-8') as f: f.write(solution_md_data) def get_source_data_mapping(): file_mapping = {} sourcde_dir = 'data_backup/it_knowledge' files = get_files_path(sourcde_dir, '.md') count = 0 title_list = [] data_dir = 'data/1.dailycode初阶/4.it_knowledge' for i in range(1, 104): exercises_dir = os.path.join(data_dir, '{}.exercises'.format(i)) solution_md_path = os.path.join(exercises_dir, 'solution.md') with open(solution_md_path, 'r') as f: solution_md_data = f.read() title = re.findall(r'^# (.*)\n', solution_md_data)[0] title = title.strip() for file in files: print(file) with open(file, 'r') as f: source_file_data = f.read() source_title = re.findall(r'## 标题\n(.*?)\n##', source_file_data, re.S)[0] source_title = source_title.strip() if title == source_title: count+=1 title_list.append(title) print(title) print(source_title) print(solution_md_path) file_mapping[file] = solution_md_path file_mapping_json = json.dumps(file_mapping, indent=2, ensure_ascii=False) with open('data_backup/it_mapping.json', 'w', encoding='utf-8') as f: f.write(file_mapping_json) def update_it_konwledge(): data_dir = 'data_backup/it_knowledge' it_mapping_path = 'data_backup/it_mapping.json' it_mapping = load_json(it_mapping_path) exist_files = it_mapping.keys() files = get_files_path(data_dir, '.md') language = 'json' for file in files: with open(file, 'r', encoding='utf-8') as f: data = f.read() file_name = file.split('/')[-1].split('.')[0] question_title = re.findall(r'## 标题\n(.*?)\n##', data, re.S)[0] question_title = question_title.strip() question_content = re.findall(r'## 描述\n(.*?)\n##', data, re.S)[0] question_content = question_content.strip() keywords = re.findall(r'## 关键词\n(.*?)\n##', data, re.S)[0] keywords = keywords.strip() keywords_list = keywords.split('；') keywords = ','.join(keywords_list) topic_link = re.findall(r'## 链接\n(.*?)\n##', data, re.S)[0] topic_link = topic_link.strip() difficulty = '简单' choice = re.findall(r'## 选项\n(.*?)\n##', data, re.S)[0] choice_list = choice.split('\n') choice_list_res = [] for tem in choice_list: if tem == '': continue else: tem = tem.strip() choice_list_res.append(tem) answer = re.findall(r'## 答案\n(.*)', data, re.S)[0] answer = answer.strip() print(file) assert answer in choice_list_res question_id = file_name choice_list_remove = [] for idx, val in enumerate(choice_list_res): if val == answer: answer_idx = idx continue else: choice_list_remove.append(val) solution_md_data = f"# {question_title}\n\n{question_content}\n\n## 答案\n\n```{language}\n{answer}\n```\n\n## 选项\n\n### A\n\n```{language}\n{choice_list_remove[0]}\n```\n\n### B\n\n```{language}\n{choice_list_remove[1]}\n```\n\n### C\n\n```{language}\n{choice_list_remove[2]}\n```" if file in exist_files: save_path = it_mapping[file] slution_path = os.path.join('/'.join(save_path.split('/')[:-1]), 'solution.json') with open(save_path, 'w', encoding='utf-8') as f: f.write(solution_md_data) solution_data = load_json(slution_path) solution_data['keywords'] = keywords solution_data['topic_link'] = topic_link dump_json(slution_path, solution_data) else: save_dir = 'data/1.dailycode初阶/4.it_knowledge' dir_list_ = os.listdir(save_dir) dir_list = [] for i in dir_list_: current_dst_dir_ = os.path.join(save_dir, i) if os.path.isdir(current_dst_dir_): dir_list.append(current_dst_dir_) number = len(dir_list) + 1 dst_dir = os.path.join(save_dir, str(number) + '.exercises') if not os.path.exists(dst_dir): os.mkdir(dst_dir) config_data = {} config_data['node_id'] = 'dailycode-' + uuid.uuid4().hex config_data['keywords'] = [] config_data['children'] =[] config_data['export'] = ["solution.json"] solution_json_data = {} solution_json_data['type'] = 'code_options' solution_json_data['author'] = 'Byeweiyang' solution_json_data['source'] = 'solution.md' solution_json_data['exercise_id'] = uuid.uuid4().hex solution_json_data['keywords'] = keywords solution_json_data['topic_link'] = topic_link solution_json_path = os.path.join(dst_dir, 'solution.json') solution_md_path = os.path.join(dst_dir, 'solution.md') config_path = os.path.join(dst_dir, 'config.json') dump_json(solution_json_path, solution_json_data) dump_json(config_path, config_data) with open(solution_md_path, 'w', encoding='utf-8') as f: f.write(solution_md_data) it_mapping[file] = solution_md_path dump_json(it_mapping_path, it_mapping) update_it_konwledge()