提交 426ec83a 编写于 作者: D DCloud_LXH

feat: 支持生成 llms

上级 7f503996
const path = require('path'); const path = require('path');
const { slugify } = require('@vuepress/shared-utils') const { slugify } = require('@vuepress/shared-utils')
const highlight = require('@vuepress/markdown/lib/highlight') const highlight = require('@vuepress/markdown/lib/highlight')
const createSidebar = require('./markdown/createSidebar')
const { simplifySlugText } = require('./utils') const { simplifySlugText } = require('./utils')
const copyOptions = require('./config/copy'); const copyOptions = require('./config/copy');
const {
translate,
header,
enhanceMd,
createSidebar,
normalizeLink,
injectJsonToMd,
createLLMSText,
} = require('@dcloudio/docs-utils')
const base = '/uni-app-x/' const base = '/uni-app-x/'
...@@ -37,7 +45,7 @@ const config = { ...@@ -37,7 +45,7 @@ const config = {
themeConfig: { themeConfig: {
titleLogo: 'https://web-ext-storage.dcloud.net.cn/uni-app-x/logo.ico', titleLogo: 'https://web-ext-storage.dcloud.net.cn/uni-app-x/logo.ico',
logo: 'https://qiniu-web-assets.dcloud.net.cn/unidoc/zh/logo.png', logo: 'https://qiniu-web-assets.dcloud.net.cn/unidoc/zh/logo.png',
sidebar: createSidebar(), sidebar: createSidebar(path.resolve(__dirname, '../'), __dirname, 'https://doc.dcloud.net.cn/uni-app-x'),
sidebarDepth: 0, sidebarDepth: 0,
nextLinks: false, nextLinks: false,
prevLinks: false, prevLinks: false,
...@@ -85,22 +93,19 @@ const config = { ...@@ -85,22 +93,19 @@ const config = {
config config
.plugin('translate') .plugin('translate')
.use(require('./markdown/translate')) .use(translate)
.end() .end()
.plugin('convert-header') .plugin('convert-header')
.use( require('./markdown/header')) .use(header)
.end() .end()
.plugin('normalize-link') .plugin('normalize-link')
.use(require('./markdown/normalizeLink')) .use(normalizeLink, [{ base }])
.end() .end()
.plugin('img-add-attrs') .plugin('enhance-md')
.use(require('./markdown/img-add-attrs')) .use(enhanceMd)
.end() .end()
.plugin('inject-json-to-md') .plugin('inject-json-to-md')
.use(require('./markdown/inject-json-to-md')) .use(injectJsonToMd, [{ jsonDirPath: path.resolve(__dirname, './utils') }])
.end()
.plugin('add-base-to-md')
.use(require('./markdown/add-base-to-md'), [{ base }])
} }
}, },
chainWebpack (config, isServer) { chainWebpack (config, isServer) {
...@@ -110,7 +115,8 @@ const config = { ...@@ -110,7 +115,8 @@ const config = {
) )
}, },
plugins: [ plugins: [
["vuepress-plugin-juejin-style-copy", copyOptions] ["vuepress-plugin-juejin-style-copy", copyOptions],
[createLLMSText]
] ]
} }
......
module.exports = function (md, { base = '/' } = {}) {
if (base !== '/') {
md.core.ruler.after('inline', 'add-base-to-md', function (state) {
state.tokens.forEach(function (blockToken) {
if (blockToken.type === 'inline' && blockToken.content && blockToken.children && blockToken.children.length) {
if (blockToken.content.indexOf('](') > -1) {
blockToken.children.forEach(function (inlineToken) {
if (inlineToken.type === 'link_open' && inlineToken.attrs && inlineToken.attrs.length) {
inlineToken.attrs.forEach(function (attr) {
if (attr[0] === 'href') {
/**
* /abc/def.md => /abc/def.md
* /abc/ => /abc/
* /abc/def => ${base}/abc/def
*/
if (attr[1].indexOf('/') === 0 && attr[1].indexOf(base) !== 0 && !attr[1].endsWith('.md') && attr[1].includes('.md#') && !attr[1].endsWith('.html') && !attr[1].endsWith('/')) {
attr[1] = base + attr[1].slice(1)
}
// abc/def.md => ./abc/def.md
if (/^\w/.test(attr[1]) && !/^\w+:/.test(attr[1])) attr[1] = './' + attr[1]
}
})
}
})
}
}
})
return false
})
}
}
function createMarkdownArray(contents = [], childrenName = 'children') {
const markdownArray = []
let itemCatch = {}
for (let index = 0; index < contents.length; index++) {
const item = contents[index];
if (itemCatch.parent) {
if (item.level === itemCatch.level) {
const child = {
...item,
parent: itemCatch.parent
};
itemCatch.parent[childrenName].push(child)
delete itemCatch.parent
itemCatch = child
continue
} else if (item.level > itemCatch.level) {
const child = {
...item,
parent: itemCatch
};
(itemCatch[childrenName] || (itemCatch[childrenName] = [])).push(child)
itemCatch = child
} else {
const parent = itemCatch.parent
delete itemCatch.parent
itemCatch = parent
index--
continue
}
} else {
if (typeof itemCatch.level === 'undefined' || item.level === itemCatch.level) {
itemCatch = item
markdownArray.push(itemCatch)
} else {
const child = {
...item,
parent: itemCatch
};
(itemCatch[childrenName] || (itemCatch[childrenName] = [])).push(child)
itemCatch = child
continue
}
}
}
// 移除最后一项 parent 节点,防止循环引用报错
(function removeParent(childs = []) {
childs.forEach(child => {
if (child.parent) delete child.parent
if (child[childrenName]) removeParent(child[childrenName])
})
})(markdownArray[markdownArray.length - 1][childrenName])
return markdownArray
}
module.exports = createMarkdownArray
\ No newline at end of file
const fs = require('fs')
const path = require('path')
const MarkdownIt = require('markdown-it')
const glob = require('glob')
const createMarkdownArray = require('./createMarkdownArray')
const { isExternal } = require('../utils')
const createSiteMap = require('./createSiteMap')
const links = []
function parseBar(tab, file, options) {
const textName = options.text || 'text'
const linkName = options.link || 'link'
const contents = []
new MarkdownIt().parse(fs.readFileSync(file, { encoding: 'utf-8' }).replace(/<!--([\s\S]*?)-->/g, '').replace(/\t/g, ' ')).forEach(token => {
if (token.type === 'inline') {
let text
let link
let config = {}
token.children.forEach(child => {
switch (child.type) {
case 'text':
text = text || child.content
break
case 'link_open':
link = link || new Map(child.attrs).get('href')
break
case 'code_inline':
try {
config = JSON.parse(child.content)
} catch (error) {}
break
default:
break
}
})
if (link && !isExternal(link)) {
if (!link.startsWith('/')) {
const linkFirstItem = link.split('/')[0]
if (tab.indexOf(linkFirstItem) === -1) {
link = path.join(tab, link).replace(/\\/g, '/')
}
}
link = path
.join(
'/',
link
.replace(/\.md\b/, '')
.replace(/\bREADME\b/, '')
.replace(/\/index/, '/')
.replace(/\?id=/, '#')
)
.replace(/\\/g, '/')
links.push(link)
}
contents.push({
level: token.level,
[textName]: text,
[linkName]: link,
...config,
})
}
})
return createMarkdownArray(contents, options.children)
}
module.exports = function () {
const sidebar = {}
// 需要反向,vuepress 在匹配路由的时候,会按照数组的顺序匹配,所以需要将最长的路径放在最前面,否则 / 路径会第一个被匹配,导致右侧栏不跟随路由变化
const tabs = glob
.GlobSync(path.resolve(__dirname, '../../**/_sidebar.md'))
.found.map(file => {
const tab = file.match(/\/docs([\s\S]+)_sidebar.md/)[1]
return tab
})
.reverse()
;(process.env.DOCS_LITE ? [] : tabs).forEach(tab => {
sidebar[tab] = parseBar(tab, path.join(__dirname, '../../', tab, '_sidebar.md'), {
text: 'title',
link: 'path',
})
})
createSiteMap(links, () => (links.length = 0))
return tabs.length ? sidebar : false
}
const fs = require('fs')
const path = require('path')
const domain = 'https://doc.dcloud.net.cn/uni-app-x'
const xmlBefore = `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
`
const xmlAfter = `\n</urlset>`
module.exports = (links, callback = () => { }) => {
const xmlItems = links.map(url => {
// abc/def/ => abc/def/index.html
if (url.endsWith('/')) url += 'index.html'
// abc/def => abc/def.html
else if (!url.endsWith('html')) url += '.html'
return ` <url>
<loc>${domain + url}</loc>
</url>`
}).join('\n')
const rootPath = path.resolve(__dirname, '../public')
const staticExists = fs.existsSync(rootPath)
if (!staticExists) fs.mkdirSync(rootPath)
fs.writeFile(
path.resolve(rootPath, 'sitemap.xml'),
xmlBefore + xmlItems + xmlAfter,
{ encoding: 'utf-8' },
callback
)
}
function parseHeader(tokens) {
tokens.forEach((t, i) => {
if (t.type === 'heading_open' && /h\d/.test(t.tag)) {
const token = tokens[i + 1]
const title = token.content
const res = title.match(/\s*(.+?)@(.+?)\s*$/)
if (res) {
token.content = res[1]
for (let i = 0, array = token.children, l = array.length; i < l; i++) {
const token = array[l - 1 - i]
if (token.type === 'text') {
const title = token.content
const res = title.match(/(.*)@.+/)
if (res) {
token.content = res[1]
break
}
}
}
}
}
})
return tokens
}
module.exports = md => {
md.parse = (function (mdParse) {
return function (src, ...array) {
return parseHeader(mdParse.bind(this)(src, ...array))
}
})(md.parse)
}
module.exports = function (md, opts) {
const oldRendererImageRule = md.renderer.rules.image
md.renderer.rules.image = function (tokens, idx, ...args) {
var token = tokens[idx]
var url = token.attrGet('src')
if (!token.attrGet('loading') && matchSrc(url)) {
token.attrPush(['loading', 'lazy'])
}
return oldRendererImageRule(tokens, idx, ...args)
}
md.core.ruler.after('inline', 'image-attrs', function (state) {
let joinTokenContent = false
let handleToken = null
for (let i = 0; i < state.tokens.length; i++) {
const blockToken = state.tokens[i]
if (blockToken.type === 'html_block') {
if (joinTokenContent && handleToken) {
handleToken.content += blockToken.content
state.tokens.splice(i, 1)
i--
} else {
if (i === state.tokens.length - 1) {
// 如果是最后一个token,直接替换
replaceHTML(blockToken, addLoadingAttr, state.env)
} else {
joinTokenContent = true
handleToken = blockToken
}
}
} else {
joinTokenContent = false
if (handleToken) {
replaceHTML(handleToken, addLoadingAttr, state.env)
handleToken = null
}
if (blockToken.type === 'inline' && blockToken.children) {
blockToken.children.forEach(function (token) {
const type = token.type
if (type === 'html_inline') {
replaceHTML(token, addLoadingAttr, state.env)
}
})
}
}
}
return false
})
}
/**
*
* @param {Record<string,any>} attribs
*/
function addLoadingAttr(attribs) {
if (matchSrc(attribs.src) && !attribs.loading) {
attribs.loading = 'lazy'
}
}
/**
*
* @param {string} [src] img src
* @returns
*/
function matchSrc(src) {
return typeof src === 'string' && ['qiniu-web-assets.dcloud.net.cn','web-ext-storage.dcloud.net.cn'].some((item) => src.includes(item))
}
function replaceNodes(nodes, replace, env, token) {
if (!nodes) return
for (let i = 0; i < nodes.length; i++) {
const node = nodes[i]
if (node.attribs) {
if (node.name === 'img' && node.attribs.src) {
replace(node.attribs)
}
}
replaceNodes(node.children, replace, env, token)
}
}
function replaceHTML(token, replace, env) {
const htmlparser = require('htmlparser2')
const serializer = require('dom-serializer')
const dom = new htmlparser.parseDocument(token.content, {
lowerCaseTags: false,
xmlMode: true,
decodeEntities: false,
})
/**
* 会将 <a/> 标签解析成 <a></a>
* 会将 <style></style> 标签解析
*/
if (!dom.firstChild || ['a', 'style'].includes(dom.firstChild.name)) return
replaceNodes(dom.children, replace, env, token)
token.content = serializer.render(dom, { encodeEntities: false, xmlMode: 'foreign' })
}
let cssJson = {}
let utsJson = {}
let utsApiJson = {}
let utsComJson = {}
let utsUnicloudApiJson = {}
let customTypeJson = {}
let vueJson = {}
let manifestJson = {}
let pagesJson = {}
let specialStringJson = {}
let pageInstanceJson = {}
try {
cssJson = require('../utils/cssJson.json')
} catch (error) {}
try {
utsJson = require('../utils/utsJson.json')
} catch (error) {}
try {
utsApiJson = require('../utils/utsApiJson.json')
} catch (error) {}
try {
utsComJson = require('../utils/utsComJson.json')
} catch (error) {}
try {
utsUnicloudApiJson = require('../utils/utsUnicloudApiJson.json')
} catch (error) {}
try {
customTypeJson = require('../utils/customTypeJson.json')
} catch (error) {}
try {
vueJson = require('../utils/vueJson.json')
} catch (error) {}
try {
manifestJson = require('../utils/manifestJson.json')
} catch (error) {}
try {
pagesJson = require('../utils/pagesJson.json')
} catch (error) {}
try {
specialStringJson = require('../utils/specialStringJson.json')
} catch (error) {}
try {
pageInstanceJson = require('../utils/pageInstanceJson.json')
} catch (error) {}
function getRegExp(key, text) {
return new RegExp(`<!--\\s*${key}.([\\w\\W]+[^\\s])\\s*-->`)
}
/**
*
* @param {string} text
* @returns {{match: RegExpMatchArray | null, json: {}, regExp: RegExp | null}
*/
const getJSON = text => {
const CSSJSONRegExp = getRegExp('CSSJSON')
let match = text.match(CSSJSONRegExp)
CSSJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: cssJson,
regExp: CSSJSONRegExp,
}
}
const UTSJSONRegExp = getRegExp('UTSJSON')
match = text.match(UTSJSONRegExp)
UTSJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: utsJson,
regExp: UTSJSONRegExp,
}
}
const UTSAPIJSONRegExp = getRegExp('UTSAPIJSON')
match = text.match(UTSAPIJSONRegExp)
UTSAPIJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: utsApiJson,
regExp: UTSAPIJSONRegExp,
}
}
const UTSCOMJSONRegExp = getRegExp('UTSCOMJSON')
match = text.match(UTSCOMJSONRegExp)
UTSCOMJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: utsComJson,
regExp: UTSCOMJSONRegExp,
}
}
const UTSUNICLOUDAPIJSONRegExp = getRegExp('UTSUNICLOUDAPIJSON')
match = text.match(UTSUNICLOUDAPIJSONRegExp)
UTSUNICLOUDAPIJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: utsUnicloudApiJson,
regExp: UTSUNICLOUDAPIJSONRegExp,
}
}
const CUSTOMTYPEJSONRegExp = getRegExp('CUSTOMTYPEJSON')
match = text.match(CUSTOMTYPEJSONRegExp)
CUSTOMTYPEJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: customTypeJson,
regExp: CUSTOMTYPEJSONRegExp,
}
}
const VUEJSONRegExp = getRegExp('VUEJSON')
match = text.match(VUEJSONRegExp)
VUEJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: vueJson,
regExp: VUEJSONRegExp,
}
}
const MANIFESTJSONRegExp = getRegExp('MANIFESTJSON')
match = text.match(MANIFESTJSONRegExp)
MANIFESTJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: manifestJson,
regExp: MANIFESTJSONRegExp,
}
}
const PAGESJSONRegExp = getRegExp('PAGESJSON')
match = text.match(PAGESJSONRegExp)
PAGESJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: pagesJson,
regExp: PAGESJSONRegExp,
}
}
const SPECIALSTRINGJSONRegExp = getRegExp('SPECIALSTRINGJSON')
match = text.match(SPECIALSTRINGJSONRegExp)
SPECIALSTRINGJSONRegExp.lastIndex = 0
if (match) {
return {
match,
json: specialStringJson,
regExp: SPECIALSTRINGJSONRegExp,
}
}
const PAGEINSTANCERegExp = getRegExp('PAGEINSTANCE')
match = text.match(PAGEINSTANCERegExp)
PAGEINSTANCERegExp.lastIndex = 0
if (match) {
return {
match,
json: pageInstanceJson,
regExp: PAGEINSTANCERegExp,
}
}
return {
match: null,
json: {},
regExp: null,
}
}
const NEWLINE_CHARACTER = /\r?\n/
module.exports = md => {
md.parse = (function (MD_PARSE) {
return function (src, ...args) {
if (src && getJSON(src).match) {
const lines = src.split(NEWLINE_CHARACTER)
for (let index = 0; index < lines.length; index++) {
const line = lines[index]
const { match, json, regExp } = getJSON(line)
if (match && regExp) {
const jsonPath = match[1]
const path = jsonPath.split('.')
let temp = json
path.forEach(key => {
if (!temp) return false
temp = temp[key]
})
if (typeof temp === 'undefined') continue
lines[index] = lines[index].replace(regExp, temp)
}
}
return MD_PARSE.bind(this)(lines.join('\n'), ...args)
}
return MD_PARSE.bind(this)(src, ...args)
}
})(md.parse)
}
const fs = require('fs')
const folderNames = []
fs.readdirSync('docs').forEach(item => {
fs.lstatSync(`docs/${item}`).isDirectory() && folderNames.push(item)
})
function isExternal(path) {
return /^[a-z]+:/i.test(path)
}
function normalizeLink(url) {
if (!url.startsWith('/') && folderNames.some(item => url.split('/')[0] === item)) {
return '/' + url
}
return url
}
module.exports = function (md) {
md.normalizeLink = (function (oldNormalizeLink) {
return function (url) {
url = isExternal(url)
? url
: normalizeLink(url)
.replace(/(\bREADME\.md\b)|(\bREADME(?!\.))/i, 'index.html') // README.md | README | readme.md | readme -> index.html
.replace(/(\bindex\.md\b)|(index(?!\.))/, 'index.html') // /index -> /index.html
.replace(/\.md\b/, '.html') // *.md -> *.html
.replace(/\?id=/, '#') // ?id= -> #
.replace(/\\/g, '/') // \ -> /
return oldNormalizeLink.bind(this)(url)
}
})(md.normalizeLink)
}
\ No newline at end of file
module.exports = md => {
md.parse = (function (mdParse) {
return function (src, ...array) {
return mdParse.bind(this)(src, ...array)
}
})(md.parse)
md.render = (function (mdRender) {
return function (src, ...array) {
return mdRender.bind(this)(src, ...array)
}
})(md.render)
}
此差异已折叠。
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
"htmlparser2": "^9.0.0" "htmlparser2": "^9.0.0"
}, },
"dependencies": { "dependencies": {
"@dcloudio/docs-utils": "^1.0.2",
"vuepress": "1.9.9", "vuepress": "1.9.9",
"vuepress-theme-uniapp-official": "1.4.35" "vuepress-theme-uniapp-official": "1.4.35"
}, },
......
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册