it_knowledge_helper.py

import os
import re
import sys
import uuid
import json
import shutil
from collections import Counter


def get_files_path(file_dir, filetype='.txt'):
    """得到文件夹下的所有.txt文件的路径
    Args:
        file_dir: 文件夹路径
        filetype: 文件后缀
    Returns:
        所有filetype类型文件的绝对路径
    """
    files_path = []
    for root, dirs, files in os.walk(file_dir):
        for file in files:
            if filetype is None or (os.path.splitext(file)[1] == filetype):
                files_path.append(os.path.join(root, file))
    return files_path


def load_json(path):
    """ load_json(path:str)->jsObject
    从指定文件读取内容，解析为 json 返回
    @param path: 文件路径
    @return: 解析后的 json
    """
    with open(path) as f:
        data = f.read()
        return json.loads(data)


def dump_json(path, data):
    """ dump_json(path:str, data:obj)->None
    从指定文件读取内容，解析为 json 返回
    @param path: 文件路径
    @param data: json 对象
    @return: None
    """
    with open(path, "w+") as df:
        df.write(json.dumps(data, indent=2, ensure_ascii=False))

def extract_it_knowledge():
    data_dir = 'data_source/it_knowledge'
    save_dir = 'data/1.dailycode初阶/4.it_knowledge'

    shutil.rmtree(save_dir)
    os.mkdir(save_dir)

    files = get_files_path(data_dir, '.md')
    language = 'json'
    for file in files:
        with open(file, 'r', encoding='utf-8') as f:
            data = f.read()
            file_name = file.split('/')[-1].split('.')[0]
            # print(file_name)
            # print(data)

            question_title = re.findall(r'## 标题\n(.*?)\n##', data, re.S)[0]
            question_title = question_title.strip()
            # print(question_title)
            question_content = re.findall(r'## 描述\n(.*?)\n##', data, re.S)[0]
            question_content = question_content.strip()
            # if question_content == []:
            #     print(file)
            keywords = re.findall(r'## 关键词\n(.*?)\n##', data, re.S)[0]
            keywords = keywords.strip()
            keywords_list = keywords.split('；')
            keywords = ','.join(keywords_list)
            # print(keywords)
            topic_link = re.findall(r'## 链接\n(.*?)\n##', data, re.S)[0]
            topic_link = topic_link.strip()
            # print(topic_link)
            difficulty = '简单'
            choice = re.findall(r'## 选项\n(.*?)\n##', data, re.S)[0]
            
            choice_list = choice.split('\n')
            choice_list_res = []
            for tem in choice_list:
                if tem == '':
                    continue
                else:
                    tem = tem.strip()
                    choice_list_res.append(tem)

            # print(choice_list_res)
            
            answer = re.findall(r'## 答案\n(.*)', data, re.S)[0]
            answer = answer.strip()
            print(file)
            assert answer in choice_list_res
            question_id = file_name
            choice_list_remove = []
            for idx, val in enumerate(choice_list_res):
                if val == answer:
                    answer_idx = idx
                    continue
                else:
                    choice_list_remove.append(val)
        
        
        config_data = {}
        config_data['node_id'] = 'dailycode-' + uuid.uuid4().hex
        config_data['keywords'] = []
        config_data['children'] =[]
        config_data['export'] = ["solution.json"]

        solution_json_data = {}
        solution_json_data['type'] = 'code_options'
        solution_json_data['author'] = 'Byeweiyang'
        solution_json_data['source'] = 'solution.md'
        solution_json_data['exercise_id'] = uuid.uuid4().hex
        solution_json_data['keywords'] = keywords
        solution_json_data['topic_link'] = topic_link



        solution_md_data = f"# {question_title}\n\n{question_content}\n\n## 答案\n\n```{language}\n{answer}\n```\n\n## 选项\n\n### A\n\n```{language}\n{choice_list_remove[0]}\n```\n\n### B\n\n```{language}\n{choice_list_remove[1]}\n```\n\n### C\n\n```{language}\n{choice_list_remove[2]}\n```"
        print(solution_md_data)
        print(config_data)
        print(solution_json_data)


        dir_list_ = os.listdir(save_dir)
        dir_list = []
        for i in dir_list_:
            current_dst_dir_ = os.path.join(save_dir, i)
            if os.path.isdir(current_dst_dir_):
                dir_list.append(current_dst_dir_)
        number = len(dir_list) + 1
        dst_dir = os.path.join(save_dir, str(number) + '.exercises')
        print(dst_dir)

        solution_json_path = os.path.join(dst_dir, 'solution.json')
        solution_md_path = os.path.join(dst_dir, 'solution.md')
        config_path = os.path.join(dst_dir, 'config.json')


        if not os.path.exists(dst_dir):
            os.makedirs(dst_dir)
        dump_json(solution_json_path, solution_json_data)
        dump_json(config_path, config_data)
        with open(solution_md_path, 'w', encoding='utf-8') as f:
            f.write(solution_md_data)


def get_source_data_mapping():

    file_mapping = {}
    sourcde_dir = 'data_source/it_knowledge'
    files = get_files_path(sourcde_dir, '.md')
    count = 0
    title_list = []
    data_dir = 'data/1.dailycode初阶/4.it_knowledge'
    for i in range(1, 104):
        exercises_dir = os.path.join(data_dir, '{}.exercises'.format(i))
        solution_md_path = os.path.join(exercises_dir, 'solution.md')
        with open(solution_md_path, 'r') as f:
            solution_md_data = f.read()
        title = re.findall(r'^# (.*)\n', solution_md_data)[0]
        title = title.strip()
        for file in files:
            print(file)
            with open(file, 'r') as f:
                source_file_data = f.read()
            source_title = re.findall(r'## 标题\n(.*?)\n##', source_file_data, re.S)[0]
            source_title = source_title.strip()
            if title == source_title:
                count+=1
                title_list.append(title)
                print(title)
                print(source_title)
                print(solution_md_path)
                file_mapping[file] = solution_md_path
    file_mapping_json = json.dumps(file_mapping, indent=2, ensure_ascii=False)
    with open('data_backup/it_mapping.json', 'w', encoding='utf-8') as f:
        f.write(file_mapping_json)


def update_it_konwledge():

    data_dir = 'data_source/it_knowledge'

    it_mapping_path = 'data_source/it_mapping.json'
    it_mapping = load_json(it_mapping_path)
    exist_files = it_mapping.keys()

    files = get_files_path(data_dir, '.md')
    language = 'json'
    for file in files:
        with open(file, 'r', encoding='utf-8') as f:
            data = f.read()

        file_name = file.split('/')[-1].split('.')[0]
        question_title = re.findall(r'## 标题\n(.*?)\n##', data, re.S)[0]
        question_title = question_title.strip()
        question_content = re.findall(r'## 描述\n(.*?)\n##', data, re.S)[0]
        question_content = question_content.strip()
        keywords = re.findall(r'## 关键词\n(.*?)\n##', data, re.S)[0]
        keywords = keywords.strip()
        keywords_list = keywords.split('；')
        keywords = ','.join(keywords_list)
        topic_link = re.findall(r'## 链接\n(.*?)\n##', data, re.S)[0]
        topic_link = topic_link.strip()
        difficulty = '简单'
        choice = re.findall(r'## 选项\n(.*?)\n##', data, re.S)[0]
        
        choice_list = choice.split('\n')
        choice_list_res = []
        for tem in choice_list:
            if tem == '':
                continue
            else:
                tem = tem.strip()
                choice_list_res.append(tem)
        answer = re.findall(r'## 答案\n(.*)', data, re.S)[0]
        answer = answer.strip()
        print(file)
        assert answer in choice_list_res
        question_id = file_name
        choice_list_remove = []
        for idx, val in enumerate(choice_list_res):
            if val == answer:
                answer_idx = idx
                continue
            else:
                choice_list_remove.append(val)

        solution_md_data = f"# {question_title}\n\n{question_content}\n\n## 答案\n\n```{language}\n{answer}\n```\n\n## 选项\n\n### A\n\n```{language}\n{choice_list_remove[0]}\n```\n\n### B\n\n```{language}\n{choice_list_remove[1]}\n```\n\n### C\n\n```{language}\n{choice_list_remove[2]}\n```"
        
        if file in exist_files:
            save_path = it_mapping[file]
            slution_path = os.path.join('/'.join(save_path.split('/')[:-1]), 'solution.json')
            with open(save_path, 'w', encoding='utf-8') as f:
                f.write(solution_md_data)
            solution_data = load_json(slution_path)
            solution_data['keywords'] = keywords
            solution_data['topic_link'] = topic_link
            dump_json(slution_path, solution_data)
        else:
            save_dir = 'data/1.dailycode初阶/4.it_knowledge'
            dir_list_ = os.listdir(save_dir)
            dir_list = []
            for i in dir_list_:
                current_dst_dir_ = os.path.join(save_dir, i)
                if os.path.isdir(current_dst_dir_):
                    dir_list.append(current_dst_dir_)
            number = len(dir_list) + 1
            dst_dir = os.path.join(save_dir, str(number) + '.exercises')
            if not os.path.exists(dst_dir):
                os.mkdir(dst_dir)

            config_data = {}
            config_data['node_id'] = 'dailycode-' + uuid.uuid4().hex
            config_data['keywords'] = []
            config_data['children'] =[]
            config_data['export'] = ["solution.json"]

            solution_json_data = {}
            solution_json_data['type'] = 'code_options'
            solution_json_data['author'] = 'Byeweiyang'
            solution_json_data['source'] = 'solution.md'
            solution_json_data['exercise_id'] = uuid.uuid4().hex
            solution_json_data['keywords'] = keywords
            solution_json_data['topic_link'] = topic_link
            
            solution_json_path = os.path.join(dst_dir, 'solution.json')
            solution_md_path = os.path.join(dst_dir, 'solution.md')
            config_path = os.path.join(dst_dir, 'config.json')

            dump_json(solution_json_path, solution_json_data)
            dump_json(config_path, config_data)
            with open(solution_md_path, 'w', encoding='utf-8') as f:
                f.write(solution_md_data)
            
            it_mapping[file] = solution_md_path

            dump_json(it_mapping_path, it_mapping)




update_it_konwledge()