提交 223f46b8 编写于 作者: M Mars Liu

upgrade dependences

上级 b6b8a5e0
...@@ -15,10 +15,9 @@ setup(name="skill-tree-parser", ...@@ -15,10 +15,9 @@ setup(name="skill-tree-parser",
author_email="liuxin@csdn.net", author_email="liuxin@csdn.net",
url="https://gitcode.net/csdn/skill_tree_parser", url="https://gitcode.net/csdn/skill_tree_parser",
license="MIT", license="MIT",
packages=["csdn", "test"], packages=["skill_tree"],
package_dir={ package_dir={
"skill_tree": "src/skill_tree", "skill_tree": "src/skill_tree"
"test": "src/tests"
}, },
install_requires=[ install_requires=[
"pyparsec", "pyparsec",
...@@ -33,5 +32,4 @@ setup(name="skill-tree-parser", ...@@ -33,5 +32,4 @@ setup(name="skill-tree-parser",
"Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3 :: Only",
"License :: OSI Approved :: MIT License" "License :: OSI Approved :: MIT License"
] ])
)
import os
from sys import version
from .tree import load_json, dump_json
def simple_list_md_load(p):
with open(p, 'r', encoding='utf-8') as f:
lines = f.readlines()
result = []
for line in lines:
item = line.strip('\n')
if item.startswith('* '):
item = item[2:]
result.append(item)
return result
class DocWalker():
def __init__(self, root) -> None:
self.root = root
def walk(self):
root = self.root
root_config_path = os.path.join(root, 'config.json')
root_config = load_json(root_config_path)
doc_path = os.path.join(root, 'doc.json')
versions = []
for version_dir in root_config['versions']:
version_full_dir = os.path.join(root, version_dir)
version_config_path = os.path.join(version_full_dir, 'config.json')
if os.path.exists(version_config_path):
version_config = load_json(version_config_path)
for benchmark in version_config['benchmarks']:
username = benchmark['user_name']
benchmark['askme'] = f'https://ask.csdn.net/new?expertName={username}'
asserts_path = os.path.join(
version_full_dir,
version_config['asserts']
)
version_config['asserts'] = load_json(asserts_path)
bug_fixes_path = os.path.join(
version_full_dir,
version_config['bugfixes']
)
version_config['bugfixes'] = simple_list_md_load(
bug_fixes_path)
features_path = os.path.join(
version_full_dir,
version_config['features']
)
parts = version_full_dir.split("/")
version_config['version'] = parts[len(parts)-1]
version_config['features'] = simple_list_md_load(features_path)
versions.append(version_config)
root_config['versions'] = versions
dump_json(doc_path, root_config, True, True)
...@@ -321,6 +321,10 @@ def parse(state): ...@@ -321,6 +321,10 @@ def parse(state):
maybe_spaces(state) maybe_spaces(state)
except ParsecError as err: except ParsecError as err:
result = Exercise(t, answer, description, options) result = Exercise(t, answer, description, options)
if tmpl is None:
tmpl = template(state)
if aop is None:
aop = aop_parser(state)
if tmpl is not None: if tmpl is not None:
result.template = tmpl result.template = tmpl
if aop is not None: if aop is not None:
......
import os
from sys import path, version
from types import new_class
from .tree import load_json, dump_json
def simple_list_md_load(p):
with open(p, 'r', encoding='utf-8') as f:
lines = f.readlines()
result = []
for line in lines:
item = line.strip('\n')
result.append(item)
return result
def simple_list_md_dump(p, lines):
with open(p, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
class ImgWalker():
def __init__(self, root) -> None:
self.root = root
def walk(self):
for base, dirs, files in os.walk(self.root):
for file in files:
if file[-3:] == '.md':
md_file = os.path.join(base, file)
md_lines = simple_list_md_load(md_file)
md_new = []
for line in md_lines:
new_line = line.replace(
'![](./', f'![](https://gitcode.net/csdn/skill_tree_opencv/-/raw/master/{base}/')
md_new.append(new_line)
md_new.append('')
simple_list_md_dump(md_file, md_new)
# import sys
# sys.exit(0)
# -*- coding: utf-8 -*-
tmpl = """
from skill_tree.tree import TreeWalker
from skill_tree.doc import DocWalker
from skill_tree.img import ImgWalker
# authors' format is {
# "name1": ["nickname1", "nickname2"],
# "name2": ["nickname3", "nickname4"],
# }
if __name__ == '__main__':
walker = TreeWalker(
"data", "opencv", "OpenCV",
ignore_keywords=True,
enable_notebook=False,
authors=$authors
)
walker.walk()
doc = DocWalker('doc')
doc.walk()
img = ImgWalker('data')
img.walk()
"""
\ No newline at end of file
import json import json
import logging import logging
import os import os
import re import subprocess
import sys import sys
import uuid import uuid
import re
import git
id_set = set() id_set = set()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -14,20 +13,32 @@ handler = logging.StreamHandler(sys.stdout) ...@@ -14,20 +13,32 @@ handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter) handler.setFormatter(formatter)
logger.addHandler(handler) logger.addHandler(handler)
repo = git.Repo(".")
def user_name(): def search_author(author_dict, username):
return repo.config_reader().get_value("user", "name") for key in author_dict:
names = author_dict[key]
if username in names:
return key
return username
def read_text(filepath): def user_name(md_file, author_dict):
with open(filepath, 'r', encoding='utf-8') as f: ret = subprocess.Popen([
return f.read() "git", "log", md_file
], stdout=subprocess.PIPE)
lines = list(map(lambda l: l.decode(), ret.stdout.readlines()))
author_lines = []
for line in lines:
if line.startswith('Author'):
author_lines.append(line.split(' ')[1])
author_nick_name = author_lines[-1]
return search_author(author_dict, author_nick_name)
def load_json(p): def load_json(p):
return json.loads(read_text(p)) with open(p, 'r', encoding="utf-8") as f:
return json.loads(f.read())
def dump_json(p, j, exist_ok=False, override=False): def dump_json(p, j, exist_ok=False, override=False):
...@@ -81,7 +92,18 @@ def check_export(base, cfg): ...@@ -81,7 +92,18 @@ def check_export(base, cfg):
class TreeWalker: class TreeWalker:
def __init__(self, root, tree_name, title=None, log=None): def __init__(
self, root,
tree_name,
title=None,
log=None,
authors=None,
enable_notebook=None,
ignore_keywords=False
):
self.ignore_keywords = ignore_keywords
self.authors = authors if authors else {}
self.enable_notebook = enable_notebook
self.name = tree_name self.name = tree_name
self.root = root self.root = root
self.title = tree_name if title is None else title self.title = tree_name if title is None else title
...@@ -93,7 +115,9 @@ class TreeWalker: ...@@ -93,7 +115,9 @@ class TreeWalker:
root_node = { root_node = {
"node_id": root["node_id"], "node_id": root["node_id"],
"keywords": root["keywords"], "keywords": root["keywords"],
"children": [] "children": [],
"keywords_must": root["keywords_must"],
"keywords_forbid": root["keywords_forbid"]
} }
self.tree[root["tree_name"]] = root_node self.tree[root["tree_name"]] = root_node
self.load_levels(root_node) self.load_levels(root_node)
...@@ -106,11 +130,13 @@ class TreeWalker: ...@@ -106,11 +130,13 @@ class TreeWalker:
for index, chapter in enumerate(level_node["children"]): for index, chapter in enumerate(level_node["children"]):
chapter_title = list(chapter.keys())[0] chapter_title = list(chapter.keys())[0]
chapter_node = list(chapter.values())[0] chapter_node = list(chapter.values())[0]
chapter_path = os.path.join(level_path, f"{index + 1}.{chapter_title}") chapter_path = os.path.join(
level_path, f"{index + 1}.{chapter_title}")
self.load_sections(chapter_path, chapter_node) self.load_sections(chapter_path, chapter_node)
for index, section_node in enumerate(chapter_node["children"]): for index, section_node in enumerate(chapter_node["children"]):
section_title = list(section_node.keys())[0] section_title = list(section_node.keys())[0]
full_path = os.path.join(chapter_path, f"{index + 1}.{section_title}") full_path = os.path.join(
chapter_path, f"{index + 1}.{section_title}")
if os.path.isdir(full_path): if os.path.isdir(full_path):
self.check_section_keywords(full_path) self.check_section_keywords(full_path)
self.ensure_exercises(full_path) self.ensure_exercises(full_path)
...@@ -146,6 +172,8 @@ class TreeWalker: ...@@ -146,6 +172,8 @@ class TreeWalker:
"node_id": config["node_id"], "node_id": config["node_id"],
"keywords": config["keywords"], "keywords": config["keywords"],
"children": [], "children": [],
"keywords_must": config["keywords_must"],
"keywords_forbid": config["keywords_forbid"]
} }
} }
...@@ -183,7 +211,7 @@ class TreeWalker: ...@@ -183,7 +211,7 @@ class TreeWalker:
posted = os.path.join(base, f"{index + 1}.{title}") posted = os.path.join(base, f"{index + 1}.{title}")
if origin != posted: if origin != posted:
self.logger.info(f"rename [{origin}] to [{posted}]") self.logger.info(f"rename [{origin}] to [{posted}]")
os.rename(origin, posted) os.rename(origin, posted)
return children return children
def ensure_chapters(self): def ensure_chapters(self):
...@@ -197,6 +225,8 @@ class TreeWalker: ...@@ -197,6 +225,8 @@ class TreeWalker:
"tree_name": self.name, "tree_name": self.name,
"keywords": [], "keywords": [],
"node_id": self.gen_node_id(), "node_id": self.gen_node_id(),
"keywords_must": [],
"keywords_forbid": []
} }
dump_json(config_path, config, exist_ok=True, override=True) dump_json(config_path, config, exist_ok=True, override=True)
else: else:
...@@ -226,7 +256,9 @@ class TreeWalker: ...@@ -226,7 +256,9 @@ class TreeWalker:
if not os.path.exists(config_path): if not os.path.exists(config_path):
config = { config = {
"node_id": self.gen_node_id(), "node_id": self.gen_node_id(),
"keywords": [] "keywords": [],
"keywords_must": [],
"keywords_forbid": []
} }
dump_json(config_path, config, exist_ok=True, override=True) dump_json(config_path, config, exist_ok=True, override=True)
else: else:
...@@ -292,6 +324,8 @@ class TreeWalker: ...@@ -292,6 +324,8 @@ class TreeWalker:
"node_id": config["node_id"], "node_id": config["node_id"],
"keywords": config["keywords"], "keywords": config["keywords"],
"children": [], "children": [],
"keywords_must": config["keywords_must"],
"keywords_forbid": config["keywords_forbid"]
} }
} }
return num, result return num, result
...@@ -303,7 +337,9 @@ class TreeWalker: ...@@ -303,7 +337,9 @@ class TreeWalker:
name: { name: {
"node_id": config["node_id"], "node_id": config["node_id"],
"keywords": config["keywords"], "keywords": config["keywords"],
"children": config.get("children", []) "children": config.get("children", []),
"keywords_must": config["keywords_must"],
"keywords_forbid": config["keywords_forbid"]
} }
} }
# if "children" in config: # if "children" in config:
...@@ -320,7 +356,8 @@ class TreeWalker: ...@@ -320,7 +356,8 @@ class TreeWalker:
continue continue
mfile = base + ".json" mfile = base + ".json"
meta_path = os.path.join(section_path, mfile) meta_path = os.path.join(section_path, mfile)
self.ensure_exercises_meta(meta_path, source) md_file = os.path.join(section_path, e)
self.ensure_exercises_meta(meta_path, source, md_file)
export = config.get("export", []) export = config.get("export", [])
if mfile not in export and self.name != "algorithm": if mfile not in export and self.name != "algorithm":
export.append(mfile) export.append(mfile)
...@@ -328,23 +365,25 @@ class TreeWalker: ...@@ -328,23 +365,25 @@ class TreeWalker:
config["export"] = export config["export"] = export
if flag: if flag:
dump_json(os.path.join(section_path, "config.json"), config, True, True) dump_json(os.path.join(section_path, "config.json"),
config, True, True)
for e in config.get("export", []): for e in config.get("export", []):
full_name = os.path.join(section_path, e) full_name = os.path.join(section_path, e)
exercise = load_json(full_name) exercise = load_json(full_name)
if not exercise.get("exercise_id") or exercise.get("exercise_id") in id_set: if "exercise_id" not in exercise or exercise.get("exercise_id") in id_set:
eid = uuid.uuid4().hex eid = uuid.uuid4().hex
exercise["exercise_id"] = eid exercise["exercise_id"] = eid
dump_json(full_name, exercise, True, True) dump_json(full_name, exercise, True, True)
else: else:
id_set.add(exercise["exercise_id"]) id_set.add(exercise["exercise_id"])
def ensure_exercises_meta(self, meta_path, source): def ensure_exercises_meta(self, meta_path, source, md_file):
_, mfile = os.path.split(meta_path) _, mfile = os.path.split(meta_path)
meta = None meta = None
if os.path.exists(meta_path): if os.path.exists(meta_path):
content = read_text(meta_path) with open(meta_path) as f:
content = f.read()
if content: if content:
meta = json.loads(content) meta = json.loads(content)
if "exercise_id" not in meta: if "exercise_id" not in meta:
...@@ -354,26 +393,31 @@ class TreeWalker: ...@@ -354,26 +393,31 @@ class TreeWalker:
if "source" not in meta: if "source" not in meta:
meta["source"] = source meta["source"] = source
if "author" not in meta: if "author" not in meta:
meta["author"] = user_name() meta["author"] = user_name(md_file, self.authors)
if "type" not in meta: if "type" not in meta:
meta["type"] = "code_options" meta["type"] = "code_options"
if meta is None:
meta = { if meta is None:
"type": "code_options", meta = {
"author": user_name(), "type": "code_options",
"source": source, "author": user_name(md_file, self.authors),
"notebook_enable": self.default_notebook(), "source": source,
"exercise_id": uuid.uuid4().hex "notebook_enable": self.default_notebook(),
} "exercise_id": uuid.uuid4().hex
}
dump_json(meta_path, meta, True, True) dump_json(meta_path, meta, True, True)
def default_notebook(self): def default_notebook(self):
if self.enable_notebook is not None:
return self.enable_notebook
if self.name in ["python", "java", "c"]: if self.name in ["python", "java", "c"]:
return True return True
else: else:
return False return False
def check_section_keywords(self, full_path): def check_section_keywords(self, full_path):
if self.ignore_keywords:
return
config = self.ensure_section_config(full_path) config = self.ensure_section_config(full_path)
if not config.get("keywords", []): if not config.get("keywords", []):
self.logger.error(f"节点 [{full_path}] 的关键字为空,请修改配置文件写入关键字") self.logger.error(f"节点 [{full_path}] 的关键字为空,请修改配置文件写入关键字")
......
import unittest
from parsec.state import BasicState
from skill_tree.excercises.market_math import processor
import skill_tree.excercises.markdown as mk
def math_processor(context): def math_processor(context):
""" math(str)->str """ math(str)->str
对文本内容预处理,将公式标记为前端可展示的 html。 对文本内容预处理,将公式标记为前端可展示的 html。
...@@ -70,3 +78,140 @@ def math_processor(context): ...@@ -70,3 +78,140 @@ def math_processor(context):
new_md.append(c) new_md.append(c)
i += 1 i += 1
return "".join(new_md) return "".join(new_md)
data = """
# Hello World
以下 `Hello World` 程序中,能够正确输出内容的是:
## 答案
```java
package app;
public class App {
public static void main(String[] args){
System.out.println("Hello World");
}
}
```
## 选项
### B
```java
package app;
public class App {
public int main(){
System.out.printf("Hello World");
return 0;
}
}
```
### C
```java
package app;
public class App {
public static void main(String[] args){
println("Hello World");
}
}
```
### D
```java
package app;
import stdout
public class App {
public int main(){
print("Hello World\n");
return 0;
}
}
```
"""
class MarkdownTestCase(unittest.TestCase):
def test_basic(self):
state = BasicState(data.strip())
doc = mk.parse(state)
self.assertEqual(doc.title, "Hello World")
self.assertEqual(len(doc.options), 3)
self.assertEqual(doc.description[0].source.strip(), """以下 `Hello World` 程序中,能够正确输出内容的是:""")
self.assertEqual(doc.answer[0].language, "java")
self.assertEqual(doc.answer[0].source, """package app;
public class App {
public static void main(String[] args){
System.out.println("Hello World");
}
}""")
self.assertEqual(doc.options[0].paras[0].language, "java")
self.assertEqual(doc.options[0].paras[0].source, """package app;
public class App {
public int main(){
System.out.printf("Hello World");
return 0;
}
}""")
self.assertEqual(doc.options[1].paras[0].language, "java")
self.assertEqual(doc.options[1].paras[0].source, """package app;
public class App {
public static void main(String[] args){
println("Hello World");
}
}""")
self.assertEqual(doc.options[2].paras[0].language, "java")
self.assertEqual(doc.options[2].paras[0].source, """package app;
import stdout
public class App {
public int main(){
print("Hello World\n");
return 0;
}
}
""")
class MathTestCase(unittest.TestCase):
def test_parse(self):
data = "$e^{pi}$"
result = processor(data)
self.assertEqual(result, math_processor(data))
def test_pack(self):
data = "ploy is $e^{pi}$ in plain text"
result = processor(data)
self.assertEqual(result, math_processor(data))
def test_simple(self):
data = "ploy is $x_0$ in plain text"
result = processor(data)
self.assertEqual(result, math_processor(data))
def test_left(self):
data = "$x_0$ at start of plain text"
result = processor(data)
self.assertEqual(result, math_processor(data))
def test_right(self):
data = "ploy is $x_0$"
result = processor(data)
self.assertEqual(result, math_processor(data))
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册