From a9143c3f3e72da1da74e0fa0b45d639311e08746 Mon Sep 17 00:00:00 2001 From: feilong Date: Fri, 24 Dec 2021 21:19:24 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 7 + README.md | 130 +++++- .../.gitkeep" | 0 .../.gitkeep" | 0 .../1.Shell/config.json" | 17 + .../1.Shell/helloworld.json" | 5 + .../1.Shell/helloworld.md" | 0 .../config.json" | 4 + .../config.json" | 4 + .../.gitkeep" | 0 .../config.json" | 4 + .../.gitkeep" | 0 .../config.json" | 4 + data/config.json | 5 + data/tree.json | 45 +++ main.py | 5 + requirement.txt | 3 + src/tree.py | 380 ++++++++++++++++++ 18 files changed, 612 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/.gitkeep" create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/.gitkeep" create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/config.json" create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.json" create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.md" create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/config.json" create mode 100644 "data/1.\350\277\220\347\273\264\345\210\235\351\230\266/config.json" create mode 100644 "data/2.\350\277\220\347\273\264\344\270\255\351\230\266/.gitkeep" create mode 100644 "data/2.\350\277\220\347\273\264\344\270\255\351\230\266/config.json" create mode 100644 "data/3.\350\277\220\347\273\264\351\253\230\351\230\266/.gitkeep" create mode 100644 "data/3.\350\277\220\347\273\264\351\253\230\351\230\266/config.json" create mode 100644 data/config.json create mode 100644 data/tree.json create mode 100644 main.py create mode 100644 requirement.txt create mode 100644 src/tree.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5ba74c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.vscode +.idea +.DS_Store +__pycache__ +*.pyc +*.zip +*.out \ No newline at end of file diff --git a/README.md b/README.md index 820ba74..c3ee2ac 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,131 @@ # skill_tree_devops -运维领域技能树 \ No newline at end of file +`运维技能树`是[技能森林](https://gitcode.net/csdn/skill_tree)的一部分。 + +## 初始化 + +```bash +pip install -r requirement.txt +``` + +## 目录结构说明 + +* 技能树`骨架文件`： + * 位置：`data/tree.json` + * 说明：该文件是执行 `python main.py` 生成的，请勿人工编辑 +* 技能树`根节点`配置文件： + * 位置：`data/config.json` + * 说明：可编辑配置关键词等字段，其中 `node_id` 字段是生成的，请勿编辑 +* 技能树`难度节点`： + * 位置：`data/xxx`，例如: `data/1.运维初阶` + * 说明： + * 每个技能树有 3 个等级，目录前的序号是必要的，用来保持文件夹目录的顺序 + * 每个目录下有一个 `config.json` 可配置关键词信息，其中 `node_id` 字段是生成的，请勿编辑 +* 技能树`章节点`： + * 位置：`data/xxx/xxx`，例如：`data/1.运维初阶/1.运维基础` + * 说明： + * 每个技能树的每个难度等级有 n 个章节，目录前的序号是必要的，用来保持文件夹目录的顺序 + * 每个目录下有一个 `config.json` 可配置关键词信息，其中 `node_id` 字段是生成的，请勿编辑 +* 技能树`知识节点`： + * 位置：`data/xxx/xxx`，例如：`data/1.运维初阶/1.运维基础/1.Shell` + * 说明： + * 每个技能树的每章有 n 个知识节点，目录前的序号是必要的，用来保持文件夹目录的顺序 + * 每个目录下有一个 `config.json` + * 其中 `node_id` 字段是生成的，请勿编辑 + * 其中 `keywords` 可配置关键字字段 + * 其中 `children` 可配置该`知识节点`下的子树结构信息，参考后面描述 + * 其中 `export` 可配置该`知识节点`下的导出习题信息，参考后面描述 + +## `知识节点` 子树信息结构 + +例如 `data/1.运维初阶/1.运维基础/1.Shell命令/config.json` 里配置对该知识节点子树信息结构，注意，创建节目录后，执行根目录的 python main.py 可以自动生成`config.json`模版文件： + +```json +{ + // ... + + "keywords": [], + "children": [ + { + "运维入门": { + "keywords": [ + "运维是做什么" + ], + "children": [] + } + } + ], +} +``` + +## `知识节点` 的导出习题选项配置编辑 + +目前我们支持使用 markdown 语法直接编辑习题和各选项。 + +首先，编辑知识节点的配置，导出习题： + +```json +{ + // ... + "export": [ + "hellowworld.json" + ] +} +``` + +然后在 `data/1.运维初阶/1.运维基础/1.Shell/` 下增加一个`helloworld.json`定义文件： + +```json +{ + "type": "code_options", + "author": "huanhuilong", + "source": "helloworld.md" +} +``` + +其中: + +* type 字段目前都固定是 `code_options`， +* source 的文件名，我们指定了一个 markdwon 文件。 + +现在我们新建一个 HelloWorld.md 并编辑为： + +````markdown +# Hello World + +以下 `Hello World` 程序中，不能正确输出"Hello,World!"的命令是？： + +## 答案 + +```bash +echo 'Hello,'+'World!' +``` + +## 选项 + +### A + +```bash +echo Hello,World! +``` + +### B + +```bash +echo "Hello,World!" +``` + +### C + +```bash +echo 'Hello,World!' +``` + +```` + +## 技能树合成 + +在根目录下执行 `python main.py` 会合成技能树文件，合成的技能树文件: `data/tree.json` + +* 合成过程中，会自动检查每个目录下 `config.json` 里的 `node_id` 是否存在，不存在则生成 +* 合成过程中，会自动检查每个知识点目录下 `config.json` 里的 `export` 里导出的习题配置，检查是否存在`exercise_id` 字段，如果不存在则生成 diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/.gitkeep" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/.gitkeep" new file mode 100644 index 0000000..e69de29 diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/.gitkeep" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/.gitkeep" new file mode 100644 index 0000000..e69de29 diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/config.json" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/config.json" new file mode 100644 index 0000000..be12a7f --- /dev/null +++ "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/config.json" @@ -0,0 +1,17 @@ +{ + "node_id": "devops-6f0f3d97e7c144baaaf1b82c7a92252f", + "keywords": [], + "children": [ + { + "运维入门": { + "keywords": [ + "运维是做什么" + ], + "children": [] + } + } + ], + "export": [ + "helloworld.json" + ] +} \ No newline at end of file diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.json" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.json" new file mode 100644 index 0000000..9720b07 --- /dev/null +++ "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.json" @@ -0,0 +1,5 @@ +{ + "type": "code_options", + "author": "huanhuilong", + "source": "helloworld.md" +} \ No newline at end of file diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.md" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/1.Shell/helloworld.md" new file mode 100644 index 0000000..e69de29 diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/config.json" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/config.json" new file mode 100644 index 0000000..788697f --- /dev/null +++ "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/1.\350\277\220\347\273\264\345\237\272\347\241\200/config.json" @@ -0,0 +1,4 @@ +{ + "node_id": "devops-00e6b52067d54d4887631d8d0c99fa67", + "keywords": [] +} \ No newline at end of file diff --git "a/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/config.json" "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/config.json" new file mode 100644 index 0000000..e96234d --- /dev/null +++ "b/data/1.\350\277\220\347\273\264\345\210\235\351\230\266/config.json" @@ -0,0 +1,4 @@ +{ + "node_id": "devops-296bbd7f39b2499bb2a30969bbeb870d", + "keywords": [] +} \ No newline at end of file diff --git "a/data/2.\350\277\220\347\273\264\344\270\255\351\230\266/.gitkeep" "b/data/2.\350\277\220\347\273\264\344\270\255\351\230\266/.gitkeep" new file mode 100644 index 0000000..e69de29 diff --git "a/data/2.\350\277\220\347\273\264\344\270\255\351\230\266/config.json" "b/data/2.\350\277\220\347\273\264\344\270\255\351\230\266/config.json" new file mode 100644 index 0000000..6d902a6 --- /dev/null +++ "b/data/2.\350\277\220\347\273\264\344\270\255\351\230\266/config.json" @@ -0,0 +1,4 @@ +{ + "node_id": "devops-2d88d2335be74101b0ae72e6540f15a8", + "keywords": [] +} \ No newline at end of file diff --git "a/data/3.\350\277\220\347\273\264\351\253\230\351\230\266/.gitkeep" "b/data/3.\350\277\220\347\273\264\351\253\230\351\230\266/.gitkeep" new file mode 100644 index 0000000..e69de29 diff --git "a/data/3.\350\277\220\347\273\264\351\253\230\351\230\266/config.json" "b/data/3.\350\277\220\347\273\264\351\253\230\351\230\266/config.json" new file mode 100644 index 0000000..31d3db1 --- /dev/null +++ "b/data/3.\350\277\220\347\273\264\351\253\230\351\230\266/config.json" @@ -0,0 +1,4 @@ +{ + "node_id": "devops-aebee081f3cb4e71975041f45a517604", + "keywords": [] +} \ No newline at end of file diff --git a/data/config.json b/data/config.json new file mode 100644 index 0000000..c6ad3c6 --- /dev/null +++ b/data/config.json @@ -0,0 +1,5 @@ +{ + "tree_name": "devops", + "keywords": [], + "node_id": "devops-0d3595ca328c49388b409e8d5ba18a9a" +} \ No newline at end of file diff --git a/data/tree.json b/data/tree.json new file mode 100644 index 0000000..0b9979a --- /dev/null +++ b/data/tree.json @@ -0,0 +1,45 @@ +{ + "devops": { + "node_id": "devops-0d3595ca328c49388b409e8d5ba18a9a", + "keywords": [], + "children": [ + { + "运维初阶": { + "node_id": "devops-296bbd7f39b2499bb2a30969bbeb870d", + "keywords": [], + "children": [ + { + "运维基础": { + "node_id": "devops-00e6b52067d54d4887631d8d0c99fa67", + "keywords": [], + "children": [ + { + "Shell": { + "node_id": "devops-6f0f3d97e7c144baaaf1b82c7a92252f", + "keywords": [], + "children": [] + } + } + ] + } + } + ] + } + }, + { + "运维中阶": { + "node_id": "devops-2d88d2335be74101b0ae72e6540f15a8", + "keywords": [], + "children": [] + } + }, + { + "运维高阶": { + "node_id": "devops-aebee081f3cb4e71975041f45a517604", + "keywords": [], + "children": [] + } + } + ] + } +} \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..eae0ff6 --- /dev/null +++ b/main.py @@ -0,0 +1,5 @@ +from src.tree import TreeWalker + +if __name__ == '__main__': + walker = TreeWalker("data", "devops", "DevOps") + walker.walk() diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..97b7e67 --- /dev/null +++ b/requirement.txt @@ -0,0 +1,3 @@ +pre_commit~=2.16.0 +GitPython~=3.1.24 +filterpy==1.4.5 \ No newline at end of file diff --git a/src/tree.py b/src/tree.py new file mode 100644 index 0000000..1b85cbf --- /dev/null +++ b/src/tree.py @@ -0,0 +1,380 @@ +import json +import logging +import os +import re +import sys +import uuid +import re +import git + +id_set = set() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) +repo = git.Repo(".") + + +def user_name(): + return repo.config_reader().get_value("user", "name") + + +def load_json(p): + with open(p, 'r') as f: + return json.loads(f.read()) + + +def dump_json(p, j, exist_ok=False, override=False): + if os.path.exists(p): + if exist_ok: + if not override: + return + else: + logger.error(f"{p} already exist") + sys.exit(0) + + with open(p, 'w+', encoding="utf8") as f: + f.write(json.dumps(j, indent=2, ensure_ascii=False)) + + +def ensure_config(path): + config_path = os.path.join(path, "config.json") + if not os.path.exists(config_path): + node = {"keywords": []} + dump_json(config_path, node, exist_ok=True, override=False) + return node + else: + return load_json(config_path) + + +def parse_no_name(d): + p = r'(\d+)\.(.*)' + m = re.search(p, d) + + try: + no = int(m.group(1)) + dir_name = m.group(2) + except: + sys.exit(0) + + return no, dir_name + + +def check_export(base, cfg): + flag = False + exports = [] + for export in cfg.get('export', []): + ecfg_path = os.path.join(base, export) + if os.path.exists(ecfg_path): + exports.append(export) + else: + flag = True + if flag: + cfg["export"] = exports + return flag + + +class TreeWalker: + def __init__(self, root, tree_name, title=None, log=None): + self.name = tree_name + self.root = root + self.title = tree_name if title is None else title + self.tree = {} + self.logger = logger if log is None else log + + def walk(self): + root = self.load_root() + root_node = { + "node_id": root["node_id"], + "keywords": root["keywords"], + "children": [] + } + self.tree[root["tree_name"]] = root_node + self.load_levels(root_node) + self.load_chapters(self.root, root_node) + for index, level in enumerate(root_node["children"]): + level_title = list(level.keys())[0] + level_node = list(level.values())[0] + level_path = os.path.join(self.root, f"{index + 1}.{level_title}") + self.load_chapters(level_path, level_node) + for index, chapter in enumerate(level_node["children"]): + chapter_title = list(chapter.keys())[0] + chapter_node = list(chapter.values())[0] + chapter_path = os.path.join( + level_path, f"{index + 1}.{chapter_title}") + self.load_sections(chapter_path, chapter_node) + for index, section_node in enumerate(chapter_node["children"]): + section_title = list(section_node.keys())[0] + full_path = os.path.join( + chapter_path, f"{index + 1}.{section_title}") + if os.path.isdir(full_path): + self.check_section_keywords(full_path) + self.ensure_exercises(full_path) + + tree_path = os.path.join(self.root, "tree.json") + dump_json(tree_path, self.tree, exist_ok=True, override=True) + return self.tree + + def sort_dir_list(self, dirs): + result = [self.extract_node_env(dir) for dir in dirs] + result.sort(key=lambda item: item[0]) + return result + + def load_levels(self, root_node): + levels = [] + for level in os.listdir(self.root): + if not os.path.isdir(level): + continue + level_path = os.path.join(self.root, level) + num, config = self.load_level_node(level_path) + levels.append((num, config)) + + levels = self.resort_children(self.root, levels) + root_node["children"] = [item[1] for item in levels] + return root_node + + def load_level_node(self, level_path): + config = self.ensure_level_config(level_path) + num, name = self.extract_node_env(level_path) + + result = { + name: { + "node_id": config["node_id"], + "keywords": config["keywords"], + "children": [], + } + } + + return num, result + + def load_chapters(self, base, level_node): + chapters = [] + for name in os.listdir(base): + full_name = os.path.join(base, name) + if os.path.isdir(full_name): + num, chapter = self.load_chapter_node(full_name) + chapters.append((num, chapter)) + + chapters = self.resort_children(base, chapters) + level_node["children"] = [item[1] for item in chapters] + return level_node + + def load_sections(self, base, chapter_node): + sections = [] + for name in os.listdir(base): + full_name = os.path.join(base, name) + if os.path.isdir(full_name): + num, section = self.load_section_node(full_name) + sections.append((num, section)) + + sections = self.resort_children(base, sections) + chapter_node["children"] = [item[1] for item in sections] + return chapter_node + + def resort_children(self, base, children): + children.sort(key=lambda item: item[0]) + for index, [number, element] in enumerate(children): + title = list(element.keys())[0] + origin = os.path.join(base, f"{number}.{title}") + posted = os.path.join(base, f"{index + 1}.{title}") + if origin != posted: + self.logger.info(f"rename [{origin}] to [{posted}]") + os.rename(origin, posted) + return children + + def ensure_chapters(self): + for subdir in os.listdir(self.root): + self.ensure_level_config(subdir) + + def load_root(self): + config_path = os.path.join(self.root, "config.json") + if not os.path.exists(config_path): + config = { + "tree_name": self.name, + "keywords": [], + "node_id": self.gen_node_id(), + } + dump_json(config_path, config, exist_ok=True, override=True) + else: + config = load_json(config_path) + flag, result = self.ensure_node_id(config) + if flag: + dump_json(config_path, result, exist_ok=True, override=True) + + return config + + def ensure_level_config(self, path): + config_path = os.path.join(path, "config.json") + if not os.path.exists(config_path): + config = { + "node_id": self.gen_node_id() + } + dump_json(config_path, config, exist_ok=True, override=True) + else: + config = load_json(config_path) + flag, result = self.ensure_node_id(config) + if flag: + dump_json(config_path, config, exist_ok=True, override=True) + return config + + def ensure_chapter_config(self, path): + config_path = os.path.join(path, "config.json") + if not os.path.exists(config_path): + config = { + "node_id": self.gen_node_id(), + "keywords": [] + } + dump_json(config_path, config, exist_ok=True, override=True) + else: + config = load_json(config_path) + flag, result = self.ensure_node_id(config) + if flag: + dump_json(config_path, config, exist_ok=True, override=True) + return config + + def ensure_section_config(self, path): + config_path = os.path.join(path, "config.json") + if not os.path.exists(config_path): + config = { + "node_id": self.gen_node_id(), + "keywords": [], + "children": [], + "export": [] + } + dump_json(config_path, config, exist_ok=True, override=True) + else: + config = load_json(config_path) + flag, result = self.ensure_node_id(config) + if flag: + dump_json(config_path, result, exist_ok=True, override=True) + return config + + def ensure_node_id(self, config): + flag = False + if "node_id" not in config or \ + not config["node_id"].startswith(f"{self.name}-") or \ + config["node_id"] in id_set: + new_id = self.gen_node_id() + id_set.add(new_id) + config["node_id"] = new_id + flag = True + + for child in config.get("children", []): + child_node = list(child.values())[0] + f, _ = self.ensure_node_id(child_node) + flag = flag or f + + return flag, config + + def gen_node_id(self): + return f"{self.name}-{uuid.uuid4().hex}" + + def extract_node_env(self, path): + try: + _, dir = os.path.split(path) + self.logger.info(path) + number, title = dir.split(".", 1) + return int(number), title + except Exception as error: + self.logger.error(f"目录 [{path}] 解析失败，结构不合法，可能是缺少序号") + # sys.exit(1) + raise error + + def load_chapter_node(self, full_name): + config = self.ensure_chapter_config(full_name) + num, name = self.extract_node_env(full_name) + result = { + name: { + "node_id": config["node_id"], + "keywords": config["keywords"], + "children": [], + } + } + return num, result + + def load_section_node(self, full_name): + config = self.ensure_section_config(full_name) + num, name = self.extract_node_env(full_name) + result = { + name: { + "node_id": config["node_id"], + "keywords": config["keywords"], + "children": config.get("children", []) + } + } + # if "children" in config: + # result["children"] = config["children"] + return num, result + + def ensure_exercises(self, section_path): + config = self.ensure_section_config(section_path) + flag = False + for e in os.listdir(section_path): + base, ext = os.path.splitext(e) + _, source = os.path.split(e) + if ext != ".md": + continue + mfile = base + ".json" + meta_path = os.path.join(section_path, mfile) + self.ensure_exercises_meta(meta_path, source) + export = config.get("export", []) + if mfile not in export and self.name != "algorithm": + export.append(mfile) + flag = True + config["export"] = export + + if flag: + dump_json(os.path.join(section_path, "config.json"), + config, True, True) + + for e in config.get("export", []): + full_name = os.path.join(section_path, e) + exercise = load_json(full_name) + if "exercise_id" not in exercise or exercise.get("exercise_id") in id_set: + eid = uuid.uuid4().hex + exercise["exercise_id"] = eid + dump_json(full_name, exercise, True, True) + else: + id_set.add(exercise["exercise_id"]) + + def ensure_exercises_meta(self, meta_path, source): + _, mfile = os.path.split(meta_path) + meta = None + if os.path.exists(meta_path): + with open(meta_path) as f: + content = f.read() + if content: + meta = json.loads(content) + if "exercise_id" not in meta: + meta["exercise_id"] = uuid.uuid4().hex + if "notebook_enable" not in meta: + meta["notebook_enable"] = self.default_notebook() + if "source" not in meta: + meta["source"] = source + if "author" not in meta: + meta["author"] = user_name() + if "type" not in meta: + meta["type"] = "code_options" + if meta is None: + meta = { + "type": "code_options", + "author": user_name(), + "source": source, + "notebook_enable": self.default_notebook(), + "exercise_id": uuid.uuid4().hex + } + dump_json(meta_path, meta, True, True) + + def default_notebook(self): + if self.name in ["python", "java", "c"]: + return True + else: + return False + + def check_section_keywords(self, full_path): + config = self.ensure_section_config(full_path) + # if not config.get("keywords", []): + # self.logger.error(f"节点 [{full_path}] 的关键字为空，请修改配置文件写入关键字") + # sys.exit(1) -- GitLab