diff --git a/.gitmodules b/.gitmodules index 8ebebc01593d0816861641fd4bad3e79bad87223..f80699d5a4a468a85da831fca5eb8c56ae7c0a09 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "build/mx-theme"] path = build/mx-theme url = https://github.com/mli/mx-theme +[submodule "build/utils"] + path = build/utils + url = https://github.com/d2l-ai/utils diff --git a/Jenkinsfile b/Jenkinsfile index 6a5de98764b103f9561a1d304c7af5822bfe20ba..2312c874d3b9c54e815ad077591e29fea5af2402 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,14 +2,15 @@ stage("Build and Publish") { node { ws('workspace/d2l-zh') { checkout scm - sh "build/build_all.sh" - sh """#!/bin/bash - set -e - if [[ ${env.BRANCH_NAME} == master ]]; then - build/upload.sh - fi - """ - } + sh "git submodule update --init" + sh "build/utils/clean_build.sh" + sh "conda env update -f build/env.yml" + sh "build/utils/build_html.sh zh" + sh "build/utils/build_pdf.sh zh" + sh "build/utils/build_pkg.sh zh" + if (env.BRANCH_NAME == 'master') { + sh "build/utils/publish_website.sh zh" + } + } } } - diff --git a/Makefile b/Makefile index b35d3f0b6a87014547a642c09f444847446e3571..345a91fdcebf77adb19111b1b69bc2ddc3a7a04a 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ all: html -build/%.ipynb: %.md build/env.yml build/md2ipynb.py $(wildcard gluonbook/*) +build/%.ipynb: %.md build/env.yml $(wildcard gluonbook/*) @mkdir -p $(@D) - cd $(@D); python ../md2ipynb.py ../../$< ../../$@ + cd $(@D); python ../utils/md2ipynb.py ../../$< ../../$@ build/%.md: %.md @mkdir -p $(@D) @@ -62,10 +62,9 @@ pdf: $(DEPS) $(OBJ) $(PDFIMG) sed -i /\\\\sphinxtablecontinued{Continued\ on\ next\ page}/d $(TEX) sed -i /{\\\\tablename\\\\\ \\\\thetable{}\ --\ continued\ from\ previous\ page}/d $(TEX) cd build/_build/latex && \ - bash ../../convert_output_svg.sh && \ + bash ../../utils/convert_output_svg.sh && \ buf_size=10000000 xelatex d2l-zh.tex && \ buf_size=10000000 xelatex d2l-zh.tex clean: rm -rf build/chapter* build/_build build/img build/data build/environment.yml build/README.md $(PKG) - diff --git a/build/build_all.sh b/build/build_all.sh deleted file mode 100755 index a014fd53e8b63b25e26d54b60ba9cb7a239f5f77..0000000000000000000000000000000000000000 --- a/build/build_all.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -set -ex - -tik=$(date +%s) - -[ -e build/data-bak ] && rm -rf build/data-bak - -# Clean build/chapter*/*ipynb and build/chapter*/*md that are no longer needed. -cd build -for ch in chapter*; do - if ! [ -e "../$ch" ]; then - rm -rf $ch - else - shopt -s nullglob - for f in $ch/*.md $ch/*.ipynb; do - base=$(basename $f) - md=${base%%.*}.md - if ! [ -e "../$ch/$md" ]; then - rm $f - fi - done - fi -done -# Clean images that are no longer needed. -shopt -s nullglob -for f in img/*.svg img/*.jpg img/*.png; do - if ! [ -e "../$f" ]; then - rm $f - fi -done -cd .. - - -git submodule update --init -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib64 - -conda env update -f build/env.yml -conda activate d2l-zh-build - -pip list -rm -rf build/_build/ - -make html - -make pdf -cp build/_build/latex/d2l-zh.pdf build/_build/html/ - -[ -e build/_build/latex/d2l-zh.aux ] && rm build/_build/latex/d2l-zh.aux -[ -e build/_build/latex/d2l-zh.idx ] && rm build/_build/latex/d2l-zh.idx - -# avoid putting data downloaded by scripts into the notebook package -mv build/data build/data-bak -make pkg -# backup build/data to avoid download the dataset each time and put the -rm -rf build/data -mv build/data-bak build/data - -# For 1.0 -cp build/_build/html/d2l-zh.zip build/_build/html/d2l-zh-1.0.zip - -# Time it -tok=$(date +%s) -runtime=$((tok-tik)) -convertsecs() { - ((h=${1}/3600)) - ((m=(${1}%3600)/60)) - ((s=${1}%60)) - printf "%02d:%02d:%02d\n" $h $m $s -} -echo $(convertsecs $runtime) diff --git a/build/convert_output_svg.sh b/build/convert_output_svg.sh deleted file mode 100644 index 066cf311f753a14ad0406ea742de7767453d8404..0000000000000000000000000000000000000000 --- a/build/convert_output_svg.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -x -set -e -for f in *.svg; do - rsvg-convert -f pdf -z 0.80 -o ${f%.svg}.pdf $f -done diff --git a/build/ipynb2mdd.sh b/build/ipynb2mdd.sh deleted file mode 100644 index 1a7a0a8f1a9c5a9b3a5e900e127a4fc48c96980d..0000000000000000000000000000000000000000 --- a/build/ipynb2mdd.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -MD="mdd" -CH="ch.md" - -[ -e $MD ] && rm -rf $MD -mkdir $MD - -# Collect files. -cp index.rst $MD/ -cp -R img $MD/ -for f in chapter*/*; do - dir=$(dirname "$f") - if [ "${f##*.}" = "md" ] || [ "${f##*.}" = "ipynb" ]; then - mkdir -p $MD/$dir - cp $f $MD/$f - fi -done - -# ipynb to md. -for f in $MD/chapter*/*ipynb; do - base=$(basename $f) - jupyter nbconvert --to markdown $f --output "${base%%.*}.md" - rm $f -done - -for f in $MD/chapter*/*md; do - dir=$(dirname "$f") - # Remove inner link. - sed -i 's/\[\([^]]*\)\]([^\)]*.md)/\1/g' $f - # Refer pdf instead of svg. - sed -i s/\\.svg/.pdf/g $f - # Refer img in the same level. - sed -i 's/\](..\/img/\](img/g' $f - if [ "$f" != "$dir/index.md" ]; then - sed -i s/#\ /##\ /g $f - fi -done - -# Convert svg to pdf. -for f in $MD/img/*svg; do - rsvg-convert -f pdf -z 0.80 -o "${f%%.*}.pdf" $f - rm $f -done - -# Concat sections in each chapter. -for f in $MD/chapter*/index.md; do - sections=$(python -c 'import mdd_utils; print(mdd_utils.get_sections())' $f) - dir=$(dirname "$f") - chapter=$dir/$CH - cat $f $sections > $chapter - perl -i -0777 -pe 's/```eval_rst[^`]+```//ge' $chapter -done - -chapters=$(python -c 'import mdd_utils; print(mdd_utils.get_chapters())' $MD/index.rst) -i=1 -for chapter in $chapters; do - # Move matplotlib plots outside. - mv $MD/$chapter/*_files $MD/ - # Move ch.md to ../ch0x.md - mv $MD/$chapter/$CH $MD/ch$(printf %02d $i).md - rm -rf $MD/$chapter - i=$((i + 1)) -done - -# Convert matplotlib-generated svg to pdf. -for f in $MD/*_files/*svg; do - rsvg-convert -f pdf -z 0.80 -o "${f%%.*}.pdf" $f - rm $f -done - -rm $MD/toc.rst - -# zip files. -[ -e "$MD.zip" ] && rm "$MD.zip" -zip -r "$MD.zip" $MD -[ -e $MD ] && rm -rf $MD diff --git a/build/lint.sh b/build/lint.sh deleted file mode 100644 index ea71fa7fab3ecf5a5b1a369579633ace84986b6c..0000000000000000000000000000000000000000 --- a/build/lint.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Prerequisite: pip install nblint - -OUT=outlint - -[ -e $OUT ] && rm $OUT - -for f in build/chapter*/*.ipynb; do - echo '===' $f - echo '===' $f >> $OUT - nblint --linter pyflakes $f >> $OUT - nblint $f >> $OUT -done - -# E302 expected 2 blank lines, found 1 -# E305 expected 2 blank lines after class or function definition, found 1 -# E402 module level import not at top of file -# E703 statement ends with a semicolon -# E741 ambiguous variable name -IGNORE=( 'E302' - 'E305' - 'E402' - 'E703' - 'E741' ) - -for ign in "${IGNORE[@]}"; do - sed -i /$ign/d $OUT -done diff --git a/build/md2ipynb.py b/build/md2ipynb.py deleted file mode 100644 index c81407fa755ef8714a6dfa14c4ecfdbb5f2ca6b8..0000000000000000000000000000000000000000 --- a/build/md2ipynb.py +++ /dev/null @@ -1,78 +0,0 @@ -import sys -import os -import time -import notedown -import nbformat - -assert len(sys.argv) == 3, 'usage: input.md output.ipynb' - -def is_ascii(character): - return ord(character) <= 128 - -def add_space_between_ascii_and_non_ascii(string): - punc = {' ', '\n', '\t', '\r', '，', '。', '？', '！', '、', - '；', '：', '“', '”', '（', '）', '【', '】', '—', - '…', '《', '》', '`', '(', ')', '[', ']', ',', '.', - '?', '!', ';', ':', '\'', '"'} - if len(string) == 0: - return '' - - ret = [] - # We don't allow space within figure cpations, such as ![](). - is_fig_caption = False - num_left_brackets = 0 - for i in range(len(string) - 1): - cur_char = string[i] - next_char = string[i + 1] - if cur_char == '[': - if i > 0 and string[i - 1] == '!': - is_fig_caption = True - else: - num_left_brackets += 1 - elif cur_char == ']': - if num_left_brackets > 0: - num_left_brackets -= 1 - else: - is_fig_caption = False - - ret.append(cur_char) - if ((is_ascii(cur_char) != is_ascii(next_char)) - and (cur_char not in punc) - and (next_char not in punc) - and not is_fig_caption): - ret.append(' ') - - ret.append(string[-1]) - return ''.join(ret) - -# timeout for each notebook, in sec -timeout = 20 * 60 - -# the files will be ingored for execution -ignore_execution = [] - -input_fn = sys.argv[1] -output_fn = sys.argv[2] - -reader = notedown.MarkdownReader(match='strict') - -do_eval = int(os.environ.get('EVAL', True)) - -# read -with open(input_fn, 'r') as f: - notebook = reader.read(f) - -for c in notebook.cells: - c.source = add_space_between_ascii_and_non_ascii(c.source) - -if do_eval and not any([i in input_fn for i in ignore_execution]): - tic = time.time() - notedown.run(notebook, timeout) - print('=== Finished evaluation in %f sec'%(time.time()-tic)) - -# write -# need to add language info to for syntax highlight -notebook['metadata'].update({'language_info':{'name':'python'}}) - -with open(output_fn, 'w') as f: - f.write(nbformat.writes(notebook)) diff --git a/build/mdd_utils.py b/build/mdd_utils.py deleted file mode 100644 index b08b9c60667981ffcc210cc512f206d319e8a0a2..0000000000000000000000000000000000000000 --- a/build/mdd_utils.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import sys - -def get_sections(): - assert len(sys.argv) == 2 - index_md = sys.argv[1] - dirname = os.path.dirname(index_md) - - start = False - sections = [] - with open(index_md) as f: - for line in f: - line = line.rstrip().lstrip() - if ':maxdepth:' in line: - start = True - continue - elif line == '```': - break - if start and len(line) > 1: - sections.append(os.path.join(dirname, line + '.md')) - return ' '.join(sections) - - -def get_chapters(): - assert len(sys.argv) == 2 - index_md = sys.argv[1] - - start = False - chapters = [] - with open(index_md) as f: - for line in f: - line = line.rstrip().lstrip() - if ':maxdepth:' in line: - start = True - continue - elif line == '```': - break - if start and len(line) > 1: - chapters.append(line.split('/')[0]) - return ' '.join(chapters) - diff --git a/build/translate.sh b/build/translate.sh deleted file mode 100644 index 21a05e0fa16c718ec54d4b2ceb83a26250a0dec9..0000000000000000000000000000000000000000 --- a/build/translate.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -pre() { - echo "Pre-processing markdown files in source lauguage."; - for f in chapter*/*.md; do - echo $f - sed -i s/\.python\ \.input/\.python-\.input/g $f - sed -i s/\.input\ \ n=/\.input-n=/g $f - sed -i s/\