From c4894b3a079df4bc605fcd2a555cf6e36474359f Mon Sep 17 00:00:00 2001 From: popcor255 Date: Wed, 14 Oct 2020 19:23:18 -0400 Subject: [PATCH] refactor sync script As a developer, it is difficult to make changes to a script and ensure that nothing is broken without tests. Test are added in this patch with 80% code coverage. The script has also been refactored to increase readability. This refactor is not a pure refactor. The sync did not work properly. There is a list of bugs that reported the script generating broken links, #160, #126, #158, #133, #134. This refactor also includes a patch that fixes these bugs. The patch redirects a user to github if there is a relative link to a file that does not exist. This is important because some links will reference a snippet of code, test, or example. These artifacts are not docs, they are not rendered properly on the site. It is better to redirect the user to the intended artifact if the file is not synced. This prevents random 404s, and preserves the original behavior of the sync script. There are other approaches to fix this bug. This includes, manually editing links, downloading the files that the broken link are referencing in the sync config file, or removing the broken links. This is the best approach because broken links are automatically fixed and do not require manual intervention. This approach causes a side effect of not requiring the config file to be up-to date because the script will redirect the user to github. We want to encourage frequent sync. So eventually, this script should become tekton-ified (tekton task) and placed into the tekton catalog repo. Sync needs to become a tekon task in order for every repo to sync docs on push event. --- .gitignore | 6 +- runtime.txt | 2 +- static/images/mental_model.png | Bin 0 -> 27803 bytes sync/README.md | 76 ++++++++- sync/requirements.txt | 7 +- sync/runtime.txt | 1 - sync/sync.py | 287 ++++++++++++++++++++++++--------- sync/test_sync.py | 223 +++++++++++++++++++++++++ 8 files changed, 517 insertions(+), 85 deletions(-) mode change 120000 => 100644 runtime.txt create mode 100644 static/images/mental_model.png create mode 100644 sync/test_sync.py diff --git a/.gitignore b/.gitignore index 204798d..01c751c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,7 @@ public/ package-lock.json node_modules/ assets/js/version-switcher.js -content/en/docs/Pipelines/ -content/en/docs/Triggers/ +content/en/docs/ +.coverage +.vscode +tkn_web_env \ No newline at end of file diff --git a/runtime.txt b/runtime.txt deleted file mode 120000 index 5f772b9..0000000 --- a/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -sync/runtime.txt \ No newline at end of file diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000..548d713 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +3.7 \ No newline at end of file diff --git a/static/images/mental_model.png b/static/images/mental_model.png new file mode 100644 index 0000000000000000000000000000000000000000..710fd44737ae47441b25123c642e0156bbe42f01 GIT binary patch literal 27803 zcmd43XH-<_)-7D-Rt#;x01Ao-3JM}1N;IH?2&hPqR7#d8IfGe55EYP28Bj8ah-47S zv6NIINy!<>6!FfD-S0VlPWS!ZaesY1nqg7Eu4g}Ct-0o$Yd<(IcV^8hrd1>oX^pg$ zEqVNsU2tEcqq>jo-2BiM z{#+8zPCIr#n~nU50=*OGE%!Y%d*)sp_4?V_YfRAtO6jlLBFCIQDp}SW%QwG{UuI&+ z^3F!=akNswvp{<;Zrkp+Z|q32>o0BZm{_q+@3 zTx)rPVireHNbl{2Sj!{Z#T!H8dwrLQZmEUq8!r6K1#ian z>-pRQW8B=_j#ulPl9XKiH$!4?YjPa4R*hA9KC`$!Ek$j`?;AJf2Jq`Uy%YQ6mK3s% zmFbt3mX1t^w3fT*JO0nx_u8tuwf^ho{_(@L66^cpdhL5!O?UW^-PFjL@r8<61(8eI zN8YUUp#}+`7dPuUM$d>`LLIJaXo{k=x3x`*+}gT--@fAJ^^%fKB}(?z47N&X*OM!L zT%WXZBt3ZWz_dpCdvSTW;OvZocx$0`@}JWoS{y#{Pj~#sBK&2XWM!vS3=AXmlnp6P zNuL-d_2>3cY+Jeh_+s+?V;NRPFjCl40-W*Fq1H6DaHpmk{GPSykTjNc=|9f#Up_IR zuHWXQlIC!``;LKH=_~xb?(#XqBQO5Al&m1-(XW9sQt|}K#OHmelje=Fe=c$CBs=!h z;=hdjZx%Qxdx-#}xw(0nKi_SoSkdd6 zj@a~VrD-}Ic=6&zUSXkjl4`(V6DB&H{@$!PG*ThF@{KI>BNnlehE&U=HhumS$_qC) zb?GzjJazKi*7r4}dSA`G$LBC=AZ*=pr>a|r7N;@VT~(-P+Zkf}PPcS(Nl8gpW4ci3 zvje*i9Pp${%m?3GvSRT;$8qC#iHX|;41Q;I`njmPyL-+-m5uG<3bhj_7Ar+F?#=%A zfoGt0QC?o&Iu2&8_*ml0eL2}Qq&d`& zmu{GKgKvb|C&5}0?o^_m<@6yVLzKqkF&2B}2X^WFr!s8&;&IW-3Y9S!?>=sB?Gm4T zc?AVeLqf`Now@|=72cfkZ#2?v8q5oK%(Sx3Ul)557M3?QFBiTsy5_90@%~OK#nyR# zxV=<8%PI8&<>3VJeArtE9)-;Rmd&tl)Up5vVV$DcoPQwP#| zQ?CD5EMn95T%IBxr;u-b)Okwa%a<>A?%W~$&csyWpw5tYKjuV2wNH$B+dpWMu` zZ{LIFnGT`Tr?;$svb=JSsAt>f7biWo@xLi>MUVcsp%EDPFU@rzYBHW~zC9 zPRL7a+&|&wDakw|;irN(7cE(Sy{CF#)>IF->ls86*2*U_l-VEaTz3i!2NrvCPKzLF zPWlJNP-OiN<`7X}A$SgqTStVd#~UW8_QncnsjpPe3grDxp@%zpDy%5!l-;qHB4qvj zvUn!vLI754uQ||gx%*S2@crB$RsGUG`KSLoTI~MsDE*IztcAu%`Sj_NlX%?HxB|_g zV%ms2J%d^Kk1tw;lkn4p$gAYEq3Ym&e%AXIZ^Yx3f5`{`_(?>_ycd(vR>~;S5bOO> zua=Q=tG)B&A2C1BP&}w!P4H$zgSzXP-`B4Eq*b$xI`kym`Qwch|0)*v}MW zD$V-*+=^3LS}(7S(6#3k4Gg&p_L8kjIyFLu?UlnBDUy9xM!u`0{^D=C%>YcN(ZRny-$o+nb$A8Whi9NmijpJ2#8F7*bB!ORKaSS9ZBE;mgG zoa6T3!H=>4e(^d@g)+a85LrXRsB0q&pESIWG90jrasVhOX}PvDw+avhLiRCD&CrFt zu;@*N{`l;|CMeOr=OnCQyg|%ibSEe0{kFC?u4Z{TIS#xI6?2^07vSb5;8WhbD6An) zBmK8EYji8aq(z7FHj<3LzFm~w_^4b@8N>(UA zuRKtg_0_DhNV}2ULb!SE)Kq=^dOJgxjNJ#b=7GWA?tOcw`!IX=q{hXIPqTO1v2$`3 zHm}dSa`P}}ezvi%a`JWCJ#4l;+t=QgPgYjbz|mCm^8M%?6r|Q_Wcr4#ys^CayJCHC z##4z-%J4RFr}UrE>~D6+->lg zr0n07ot}Q*?8YrFef?Kh?EQZaQ2Zq~7A-+KKAAMA7OOOnHEnv>tmzywI?}%A?d5BR zjj}wMg`PfpR`6u^8K#U(X>X1%$!6q^@}NRQlg_CuiO~3MN=}D*OLTn6UfkA$O(AC4 zv(Gki${o}x`rR~R=(HX+v_IJ?%bz!?ytJV9E4TQv(!tJ!BVg)(@ahTAW_c;8=j3b` z&)_#{--ZRN8*Q7*@2uPyHOt0dI^t_#oyl_Y-s%_Dy|tdf5nN=dUtou>l$0Cw1FzN{ zj-(dvMpONs>c|GGDp{wAM%%vnWG~=`4l0FOqf9PuEB3xt_ihjV+pm%4vt-5E$4{Od zf4KQTQALGbOWuR+5-!_sU|S>@d|`X}>J`0l(52AKby{uv+7++Pcy(k=59Dek;7@YS zusSdO7;)iMS97+6T)1~*y6s5IL*9$3s;YaMk@vf+BR5{XdiA|&W1x&551&!(e!1Wy z$FcFH<>fiKxZLq(f==;<@xD5R1FXlWp<=~FMb8zZm1f4`UE49v*bltIR$a#&9E1}R z64W#7o=D8oseNA)ZhLsFX4}0xK*)>*V664_I`M;j^t4xzF_YDa`tlrd!A0$*ewVXd z#CD%~q|=aM-qzXa?eAYSIn=sx&6@VEFR%8iyuI=C=~J%wLT`ET?r%zlhFqBrV=qtM zU%OxJ!)g++SBCvaO81+fz5Dm;043o9G~vPc_W*&JAYbovG1or^z4p8=OYUB~P77up821ar${YE^EpV6O{XOkBq28>@N^4 z96NdGA@IiM4<>5&)-V?UpGZkdzle!heIwPf6No4mlTR-@j>x%FoZwm~462*PPuv#t^4l`V4nB@^Q(!w@!2m7U$6mq=A&ywNvB$GODUfk&%(_ zuT{M%sL}(d)68@rb_9!KA79pF3qyg&wk03=siL{eW*IqvOP8~pjspFX1Uu3@0*$0( zW!v>7rnhMqKH4KHno)9O*REX%yY~@wety5b?vQ22Ibw1~I?4kM>8=8>?DWf?c^#d@ zH0Ja=IA81PxA)gX=VwMdwjX^O5wTZGTN{A&6$V=teAN4Lq8^{qVA&I;CpE;Dv(S;fnUs{ui7<=0z{{qf$ouCg;eQtB>8jgDDr z`{oZO_4jgre6NmFAl95jjDnSwT5p!U=HrW#_m5cjY|==x8ZwxE{ra`>xA!Y=$V<%n z95(qz`kZcScG$E*>cWNfy1Kf=rfRV73=uTd9&F0QHduaRo%rNO->Wl|Lva&L*f%GM z{Z&4coIShd1`&XsKHY%1jGl4C`v&zN4h&|DtR<11#~8`3vx1Fj)<&b9+`qH3%G=qc z^zQ!{WR~@yTyIH<*Y$NQqEecgEF{!iogeB=bG3&0&e%3*IXlgc(T>rLYVem{owMWp z^^5lI-TNKe&uX9{AW@%+Q}*f0t22ux+q~rsLxPgZ{ZRU(*$!5$5o-8IAL$GbcNRW% z>XfxOYj{;LCgsLd=Lq(&`k78f{kNbd^70>U*|&dx*r|t`#|9*x+h}$A>R!_w2p7g5{%L;#HvVnLTRi>U~qL;Z+a7W(*G^^eB!m z`B;~zf2}DaRYX4VY}TqZ&RG|oH8*eClp+^fn3wlCR!s<}x^-$IaPE4PysYJ?DtmHO z!|IhQuVM#{_S7)kh=_>b(s$yDHp#+PRZvnY@Y$!}=IOadM5NxDD~+ewbzbcJgp;^M z+iwWUiUpZ=Loa)c?&n_U?>w337kJ7002f!31yhts{Ra~2-HYn#WlPqHp6}-iJscW{ zH-=s3CtPDxlcJiwm9*lPhGr+4tFR_-p7-;+IKYcnuoGc8`N!(OpB#WawNC?Fi>*QcV#`HjX z>PUr1z`?7F@hPLdG%p{YO6`yB|4i%5$B7M$jHjffS2jCO z?AZM9E{>2^f(o`oG(Xt{=}P1gtHgjq7NRUW2IxFJUPC{!JJjrX_m@}qsmb<;9Q!vj z2sJzLlaK=1QQwmKz9l^bblbqny6zYfmVHlT_>DC3I#gZ43>SmaI-F%Oq;)=&(9H2~ zMn~;_?!+W@eofHc#K^cFLpgia-H0bmgiIuP;SJ*EElUyMCc3QbPIe|rX&YRW9Y=(E z`To5R!r?*Pl1(_$T~npD2&t>LA9;)vI20~1`vCWal|F~F&gweDXGDu@2Wc1aE)&6^ zdQ7@BEn|Le+PFC@#E`Aj?XxO)0>)_8l#eM zi}>|*5;Ko64?~|Ac&h{AdHD6pzM~e_&v$1$M-dMp&ri`5J3iv~eU&wTAf=*6`)(zSGpaK;Lnjg|M$d;^`e53)s(RLe*fK=i$CA_@ zwK2hV;*<|%=jMJ$Y?DeE-V5i?e|&a84QD~Oy~LOJdx^OrFT_JW(}p9-ou%%K+>>=w zmjc|1U4IhoYnnAT#dR(<$Z;^6np;KRU%B;!Y08h~`uV4l#+=#fh+^t=TuUxflWF{i zG*putHg4=Kzj(P_;VdI1!@2vlpXj%H7mzeh1IumOwr%c_)+*(!EIof$1nG0ZN7e_G z->d3o=jZqC>oZnv#C?l}dMm#}{>FVYSW?B*Q?BPko6t*u^e`jsKvHv$T9gOoDz14n zyRL~<$SNqOg+0=7oERv_(7`o?BU>%RXw8jX@@Xke4Or`<*D`()D*O5q3R$%4KbVBQ zdE+p)C=lyXXMI_Aer~q7w3J`=cI@-ieJw{pBi-utyPf+ZBJnHk&efH3*RCAS`VpOm zG$-Vn28Z6*jJc^^A79@DllsHiJXDUc((>{#+dIOFGf9EZo}C7b#hej>Pkb60S^=#f zE71G+x$uX+)G8<0o((MP&Gv9|Zrr?C>2<1}uI?gF*W0A^eAyghp~l+v8TN@rip=~5 zQlSK&X5Dk?_J(Dl;h_$zKqBFK($dm#%j^APl%kD(%z^-NkjSk6${`ylDJQqqw!wUb z+6UtyyNa68$wxaxdeV$mN`YND)-Hcg=DPiOo&K!L(Tn+wCWdMGA=@rB9Oxv77#v#b z0~>qFTb&d3dbmkM6gzbNxX;=V6&=@-+8G)>J|;4lq0nBjlkvpKlL5lkT%fq}wG!UC z&bu5sbSP^7;PUn}fv%IetWsyszQoUF`psM|8$H9`h$L*&$o-*zNE=J28a@l1GIXFu+OfO_s*-RNQKLVM2Tg^#CRZy z4o8z`-N`d8TjF}{jM6tURJ#-gxr?Olzx?jbkKFr~9c9~Ge%@v!Ri|bmdJ|kC&>J9m zI5w^4+s-_V(+z9(fe}L`dq^wqx6@uM<$%~loOc3~?wTpYDp-_&h$ED&)#;tFd7l#MKb z)<=++QcPXWKS|!TFfL(@K5a(Z<&iO3EA!UxK7A0N6k)4`fj8rPtJV@= zNvO6d4C?CY;!0*nx9WasRBvsfj2+#Ou5aR~T zSz;tqYBOz|5|I;pI8VALvyRbC%H={uyT5F({$mi^k6PR*_2W$+Fg*l7O=JBH%k%N( z681fNt|rzQ0F@zQ*@%P-tR*+E>88xja3aODuaKBrbm78Pl+4@jOlqn%hc*)8w-?Rmf5aE4^r$N!K@LaVoRXHGQj z-m~YfvZnoA)E7fIIJJ(J6QF1byz?!|2y_Xt6L5;s#q`=m67Jl;uW<3=rxB{eI-lmD zHyBmM%y8*fl3izR?p$UMzyj-ZcEX}(2h|A0o zfR3@=FUGR*_6nh{=f?&_kcq8gt&_?<7nZL#F8vYG25w6w$&hiyIu>?z_B)`TpW*Tw zH*EOSl<8=X9LD5B$AJT^e*l^s$fV+pY>+zZn_-`HYUy4qWr@C12q*o0%uD}gm;DrD zqARx8{#dEx8_$~E{!DE$pVuWv+e@sY^HS)SckU&f*TA`NUZ0x}dBNMe)gfB<#;Ya*IEx^Iy=E ztV7$Xp8VhSy8nxo^ph{wU0yZd9G6ia!4nhdY0J;#w1v#J^a+;$=c%w0$Koe_{QXth zd)>45H`Hk&q1)TrTi2$fi;-zH&v=D|l1nTPt7R2zr8Nq>#%}fY@zMDfFg(0`5z(mZ5h73`fVvCv?JC8ZFh6nUSUPfpAuh!qGsc#5MQ2uDXohIDe28dT`2>#%X;Sc9n@|A zU3(p2|5LB~N457agZx8i>`sy0MQ$Gq;3Mmn{FOw!ssATq8#`HDM2L*QV1F{zgyPHN z|L5lU%Z2}uO#a$ulTKBxAT`H*ec>c%mg_-hIuti-FZ8dk8nHNNj$wiNq4ovPSZ>`? zax!d#=vd`04}FuI+iPKIDShso)hx}J{;^?toe#DC7iGS&_sgplM~@x_xv5Ndv9LIV z*H(i~jq=6pDvG8tjGju8Ocfo2q>Jf52wf^6@5)!1Y7Wgn~w=2&8xBaj2d z7`VExug}3!>WYl4to3YmT?hGO?@YiT>Vcu|_9G}V6g797jif59aB%kJy}ob&)Qt|- zCf~Z7ZcSRba%Ee4J6*@ox~6+7^SdutV%@a?=5vB!6=b?cN>fcpufc6y@czSs?8E^P zSah0a?~4h8Q-HsAp+K9aZ*zNMY?IzN zwnf#sA@=sL0@qj#Ghf_NzmoXOx=u>=hzVw2V9kdUr(X5fC!1v^%{93W)ZV>w2O6nr zyLob>4f95hbKg;CdTixe{4l>|$~L;DXUYL|k4mf(i_7Fy)epwSGQ632`V{Aa?76Y{ z(~c~R0fFDB_v3p$COS=h_m=v-sJNI<<96(0e}5@CxkuUBTMN3ahOt2`kn$aj%+mW0 zLoHqL5iAS=wo;J}Js<6L(TUDTyDP%2k)qal{fJYVn0ybrUG->Skn^0NjW|C`j%Km< zghBb-lyHdLSs59PF%V*qwUy~vOR8h%K^0yDS{s^cT(0{=lIAE zVkHmqXRc7AT$w(2$%Ko^;4Wyq)Vb`pZy$rIu$=sP)$7@i_s7=|*A{X8&a9-6@@O2q z+04)T5=(dP-|s;IYKe2LhkSi|#k%ESafTj}XKu-b2-EifiC}M#Ho0L+-wJ)3Xma$b zth}gc-_D4_y!<|^i(CH1x#)3TS2}}DdI$>FCm|u8<$A3EpUISs51&}2sjz=sYH>UE z_oDDeE7mYwYVXA|DyOdQ1h@hfSZ(L`3_Ad|;EI`vs&GGE^5KKAlR=0;i>00)eiOKK zdVTC~kVlt#TrqshSZ%Q9E3t2HTrpnw#-ILQxz(d&%Wt$Chf;6f7dull$z;!HrzTIM zON7GV{}mr|J1a-&w{hPWVb?0%<+TiD?eoLmTwm(A$5z66`|9}k^?w*S?4J@!8po!x z!JOJNgI~@bge1uq@TTDkcf^;oZ_neK^@3mLFTV1<@|=TPYlGq2n#eC_pBo*LwLxJB z!nfdUzir>wCzy}`8^;!As9Xni(gzNIC04Fu2+GLnVvF`80}-O+`|DUMuo%5D;2+cL z>y=;m9;?*}G(H%mVwDl;VYD2Djy}LUD%>gKpwR6(AW41TGFW`@>Zhd?-3X)Y^08{k zkD6x(Y^`foZ##4cTQ5CRR#0{2rQ?pbZ>X7N`oaUya{Wk@hNCU&CqBKht;UC$VTTX} zX@rtM(`o1w%_O3-s!Cc)>iR638c-bbVdE>Hk7o$&9c7G7|2KmF^i(E0CzXt!1m={H zX~m&k{2eH4|F2@KD%DaS*LdLW-ctEmhA5Q89LGsNQGy*b_`I_sRIC(bm7$X+hOmPu z`=aBsUP5{iottcB=ioR5$`|#_QBYY7oA*C|{`|&&QU4iJJ<--3^;ym$>qLK&2s=ky zJPO^*CJkX>>;<08XTVB3j`v;t#nd|lb3P`N8QlBuS8Xpon?4ojdBqhKra#9;#ZJu> zFq<7jWftl(bxrlX!Eqa#!w{D8Oq2QThjkDEP+_lxw$cu04lBtMoT3pxWD68P8yOk% zTk;>;gE&aVh(KztlXiyy2gvSW8`EOV@CQVYG4(h2G?O0e~N&h1n%`YI{c zcjew+r;id9Rr`7HHmFonfO)%WqPOf2wz^XHPM2UwOvNUSWkuc)(110}w5UWTOFbdC zF>MuSIVcGUS8^7MPj&5HzK*4Aoy*T9q?VQze^L8Gfbs399|`u0AfH6XziJ-|kt1r; zq1M7w@EePuW4s1o22S{(>zot3I0Sv2JyjEL(3mC|tsEa`qnZqJ36^6RCXb7N`=Fh` z<8R8IpB=RC2+$`mIkS*yTd3o}E-|qXq7p6_q#c9~bJ}C;@#DuA!4MKNlzMdg+LqSV z;*ydU*7?a5*zr0YWdY~>4-)o+d)V#B`~Kh%cAxgR0%CAKIV-rRNa~GT$XOj7c8Ew( zf@G7p?a4SgVd3HYSU5uRI(>TiIhdTV2Ie_T%qp$6Scxc=h=tT64uFE#N4f&wInLy&xkK7^78NsgHM;-`CN zhP+&Lz(R}8j#hc>5P1fkb#87VI}xf5ja5-O%5Dbtb7pR)Q^FF0;0WfZwoZpzz^p0w z!mHEfi1Z2|m2tF*)vT+l8y#$>K-w7Y4yQPbW^gn0gE?ZFop`vdgRDU~0A39*Yk$%Q zkxrJ73by1B^*T<_&Ebh#UwB|G9Qf3nDAhg2t^Dh z5t>*cRh;1?QJ!&9oLZ)Ildn~AKv^v?q?UJEp?*`}<}lt@oO64HMskfR)Ig-pk4Ww3 z&mZ$*5&N2I$$5Y64lSr|V;#}5vaKvfuHS&4hy+#YCX|_h^uh5X@){7pHO)LF6lb1` zO?AB@if}}Xud#))p`vyAXLR68gVAgRbj*@=eGcTDQ%LxeU{a6T4s2hsc01`i z5k=O!oUdNpbwKrfL{jl$P^Ld|AQ%4x1WTXQ?7=+$wsPeQv*s-0#x!4WZ+0UB8XztS zagm5x@*g;p>a$%X+1g9rKUW{xNAM3{zJS$JU-KH`h!YmDX6^t7$30B-=OiO# z9&%P0(98n!4+d}_7njz=Kx16tC1Zw?ATKBzhg^(}W1@F0!R$Z7>0d%9ODuNZ7a_bV zMc0hY=~-C`V1c5C{l5i^*sg_EaO~W<)sTp^K`Sc`=Pww?WG`Rdrts#RxpSsPdx`K! zO?!zs_h2i-79Ii29hcBc_j?@`qHvZ*47r5*rlx0d_Z~L>itC*79n{s+d#6{a{4y9E`5*kXu%gj0@Cx)45zPeDdYv>(w+E|hSd zdf)4J&}V*wC>T}jUp?$4QS6eq-M8HJpnn~G`SPW@*R?M%PsP^98(1+?^|1oy0uH?{ z_=dtCf{B}#mp+1~6CyxT$C9Yvfz@l)>=6)nUtmrRKC00En>aLbU?IW1R8Yg0do2Zn z+S+vB)`}9K;Q}8=m@iNEh!tR@?hhZ@f4{jRYL+02mFvfA5!?u}G@;HI668JdY>)D} zP|?(!`slCZITuZ~fmJ154_JPYIaSfhyrku8w|5z`h}aH7(zTIOokW`abj`W;?|LCUIPka zIuemPKjr3vn*D*~yy6PP8sr%7&`>#?La2K!;I}#-@@7Twv3{h?jdF(CcWeW_BxhU%ew5NIg{5Vuqkw3b0%X6anX4smWjR)4hnFtdfE+NDc*g{-b+AkqSk zn6p9axiZ{V37XkQQWQHc;VG7ub|4xZjd_xRV_9kB6Sb1;Z z^U)3k&{(ock_?@5;#_*8pcOdDBJi2R0UO@SYW=77g312 zYRe`{VIK*mSf&^$MdJM!K4#*zHSs!*Z{hv&f3s?7wYke@g}Ulh=(kzF&r&TeE}N!$ z3;!19q^B{o@BBEWygxS3CtKpwz72c1lkkH-74!R zus#CEn*>yyi=+v@gH86*Nn(%$EpjUGp!~w?PU7_u+l9gB;t!-kh@m<`(i(Co4w@n`h(h8_dFJgvg(raW3g>vv&!jLN;1vqm&xMb^uhAF!0TF58q6kVBJeeCJJIy zZWr@`g}dt#IbmQLwcYgO2DlG4MIw}c{(K%HaqHk-wng$VxvIlu{X{r>^Li>q?^f#1 zr>#y2d|%%$9LPBI6f_zsi8ZOm;Ey}Hk)U0OQi{<3Na6C7f*;qr43zR1FgGWVm{MWT zrix8$M|dHHg@yeDiD5p}atb!}ADwsDZWe?4Vry|tw;g-~Q(8?{HgX^#v)q?t%gF)Q zyisGgEMnr5WAL?c@*96K zL+9r{{UU}+Cdj_o`UOs>A4NNP`mH-?EuJ`-eWir<6?!k>wsnw4V+aW`_CM*m(Z0GW z`QpWXO+>Q@iO4O8&)yueEl_=yUdg?;=4-zV&O$PyC^9=W35~&PP`e(#?N0hw&*$XWcS`$V)EE7p0csA z@r}3MRDXTp+j3NgXRlq0e594B^WS|UCadWGnw0R0Mu-0FZ_7?*B^l8yP>#ICu!W71 z=6jk#d46CB3Is*7TObWEvc7pelOi0L^2z!H7n9DNlcMm8iy_ngW`~JU>Uw8V=K&)d zqr3?>X7337|EZDWul#OeCt(wozx6~UK9O_e-wnHQ>1ZA)hBugDK-hIaVx=hr0ZB>r zW5mwKQC0DJlAb!$=E7s1`I=IA(7G_K;v}s{_@FLN^S> z)q-`_n&>~F#Ct5x(s=SZvSS)$1%wUWu5U@)SoW6@5pJ+$!it}RK8;DRb*;5uWv%D; zkrn4uhaw{}#B=!f0UdI$3^Fo)QS$U@2m;6P4R6NJTiLH3xJik){pKocv%}aZ{WTXT>-y`$vyt?}8W9H_3D6WSr*|3iH z1C2?OwCuTjAhYP%Y&SRmpg`jrMesX@h223)UT7H*s+-yIoC8@kb~p*gsf2o)mft9R z8&7;f^#*mb9^y@c9mik(j`z`dJ zoNl|_GhKa-j@`$k099soYD^ovQc~@9%o)^iiia&k-i=$foH=`T72(esRNMcIi#&az zyc$6yy$rTcb^G;iztcw~{;BO!4oHL3b zlxp)9NuuU~C2!kN3-;Cm&)sMcffiig#VU_DaK?*8H|N$elwNQD&D9F~=jB!#^raMm z%h-C@cq^hr>o=o#pm3kbNjot2V0&<(>=JE)q(Q+*kQL~M!O}?qJW5@5*!(7a*ug2|LY!Se{!QrMo@~Dj`Q=#tP%ddP(rh-3w_53;MqxB}$7YgBW z9?U|IMn|n^7h{v4^FsC@oJPlRyLBj$&pg_`=n59U;d<^GoaV)Z$pf~5z1-Yb#pW^f zY}6!PP0g`pu9Jqr1zO)^>?9>`ox<)s>NpD~=+MX2x$m ztj-R@D^5C3=PI`fY$i_JZAM1M)JK!gGp^pfcW*JQ8%5BK;DL4jaa6A&xLDqGN`Yvb zg5SE{p_iK?$FqNbjD^r!$mQrQfggGFT0ubrt7ndr30ac82#Wkt*p)$oto{f(Gu4TH zhy-irOE5)58TT2yW~f8YzUMPiu@;u&wYJU9g&;WTlzBR7`LzpHqhGBM6SH^UzMO8Wix~P0edO}nD~qs_ z1gC)xt0g}{7*$3oayL1R$im;;=DnY=W(>9Di%$0?iq8DZH5DpG3@555uwpm{Fq;S4 zk95%4t9+@6A7>@>%_GvK|wNbc4lh39V;Kn@AE727o&(s+?F+;Cu9=N>>$)) zgIUu&gne)ekNQssP}KwDEb<+c1w=jT$4Pr|nXp-8W)7SS`{Zj`{AYY3QW%w=H+$v~ z(L;#%M6g}p_XJfoT69{E>&y{QUuw5Xn$*&3hYj#wWxLEW zS(m}Ah`7d_+zWS-1DYzAO>)M&xux)S1sVgn-DVAX{P@b$8*4}}m6esG!4cK{bN2LE z!8kW5m@|H*+yY3tS8ltOJ%0d{cK)c0Rr*2tXSlwW4?@b{SZ6#o-vp1)QOF zw}6a4hs(1P9&+3h(OpO|tjZeJj6i0(V1%%wgr^vX%O3rL=!IEE^d7Ee5~|<#!P*D>ry-2s7T}S(I@22O{q(4P(^dXpT30j*7LIT$= zCW%6Yc!4P-S)sE8*Cmt@8jbLu65c#GXZ9R8;8|UL2?9&*^W(SG>nFiGml6Ij04z%) zNg_eu6GaH-yzBD|Lev3mf|lAghjS-K5Ej9%6TPNLk#-?o%<#Sekn2E2_h1nV1cM3# zl=Oet%&uTLt+Py(AJrqUfMh1zxGC_yKJf5pgAL7M+8%J`)(zF6eLOsm2>%<}z!$x> zXg^B`nK#hM3l=6W4LPjIW!eOxuosGzRd?lG@Tqs;fCI<1NS#cMM4*Ck<}u;`iO4@_ zwn5(pe3hv^T!DQ^xm}g783+cR@caQd{D2BWcwT3Q{jznCav&}*BK86wl$j>G)-^O5 z3EAjJDQ|Wdjs{`U{=@o4d&29J&8fE0b_8N6Vi;)0*a>s&ZtS?*A^P`k8vW@75V5uB z`n(INc~0Yy>?WaioSvMivo!61po?2_o(&kjYelTIijqQf;f!jO;1rfxbTI=D4 za)uP95D~9Z80F!B#)br_E`WIPi5TvK2j(^Kh@{M&hhrda@czAf8=(5Prs!7$?>c<= zLy5Qnynkn?CnfXx7WatNA#h^NrRC(7Ux6Y=Y{D2NTi{8e!Ch>kVTq5>O*oGUA3`9+ zca;8nxwzP{me}wB5;umIHu8YSWSYUqct=&1qel&DFgaH43 zFs?g+9H9a=g|OM8VPpxoN+JfwXl4c?{SBdTBvy%Vy5CyHK$y0Xt2!yJSp^B=bCcST zt<+`5k=JMX^d&5vXK`q%koz=U=(^w&nq8(3K=vh-NW2C{=F@Z@Q$%x`J*?(vg(12_ zvABsg{i?xo3xX5S24RD=@SFxxI=t6R_wL=hN#dseMCN`+HKGgW4A6F}5EbYfdHxTi_X)2U+}BRX@-8VKWGHc0P=YS*PH z3Z2qfNa5ZA0oT4cWo@TL^gdtGBL*n^6dm&-qAySj!t4ND`>5Nvffx)PHL zZU5DPY3KXuMI3Ms_772wF=cq-kMLr-7fb8IPBG)J2hnu+5e5(Dj5XvGi@0T_j#_gi zD{3Yp1BTHjybKn97naZt{^>c?V^{D13iLOAGiz1ON|ZZEwDz|aPEOlDbX%IvG2h*S zObPk+6!3t`2jgvMMb?b9^^XLnag~q-?%<5GvOy1j-4-L2xMEk`qq&M^_M4q-^KN9?|L&`rkv#*7gm-mXS*Zo;;Liea`~ic>G9lj%|hX_{dPbd3aRRSzx#tEcJ{Hle?#<;D>=FHw@M@dOb9v70!+g zJM;LJn)$KtGbQGpGdUSmetw#!gE?oyoU8Nr!AJO{H}Ap5DTvepFnXt|^PqL=9JC@A zhIZ&}O=dH|p6F>7`)=O5>iv=ndn*W65ARLLB6ju16x+_-S2oCzHr4 zZam}R3XGxkDxNxZ7Y%@MW?A%tNvC+tOrA7Pjh#1b{P}k8+b0de>>G>W@464fL@L@fZGvQU+-i!Mfozm0S?~QkzjfQjO=pJ_VE^yxtZBNb+ zAHLkVAK5t^jo6n6$`Yxr^7~qTJz&v&5)x$G`curfjm-yiq{IW@89~z~o?XHjsV9>{ z?CPdotY;j;iz-lP4RCvOh=?s0c-SsjLIz!XA@tHvvo|9)`!;XOY6A@ZL3GQq@!o-IP}((tdaM zY@X$$og2RBEF!&KDZ80O3i|x7mv`Xz`}TVIqUgYPE#7s#yp{9LcTn3e%m(j8WhcXZ zBZVRRNuA4*h z9o_3Z2LtiSlY!5?aC<)!O&f(<ywmNOeK;Zx(RnR(aHI&`Pc?%2-ktlAUyv!|xA*NF^Ek97(>9X>TRbLwOk znjg;thE>4;y7lcdI)js_@`Vde5|(SfW!zs@D=j#G?4i4Rpi{=o%*=CbQj&{v?B&e4 zz8IsG^sp&QCTaR7L07a&tqDBv4GmZV4eJAmaPT>>K;3mcUA&G=(Gi&4y@x!~wxmz1 z!}XIvKwnB~D=`!QUn^H{XC!%0uQx1EsAYNdy0Mftqfp@oPD(vtMO6Jaq4bjihen)D z%f^1~5%U^n?d&=sdQjvD-D#JYW<)PG$|I-R9yT#@811g*-0zJioM-=++vXFVv7xT6 zE>*aPl4M@2Z_;2;^6L{<)ws(i(O7?t9^!_*dZIP<%9*ULIPHASuXhzig0S9Z26Bx_Q%36LN6Bz3w0Cq65z+zrP7N?Rhv%S8`WRFiL$IfKLWRO!tuvdH zQ$v{%aC1yvx0sou3TEPq}Aj8z4xKLD7)OBjYQ`4LqE@CHq ze%_wnQBXk9WwJ9`@Zlw)X>2KOutnv;5} zvl~+oc*OiA7&Jy^W^UXIxMxjNn^lD6L&JsM4w@CEpnBKOl&vJ=@7IsK$I~Rl zK>7pq-V5W{Io{Iv15)qEIXtxj zsob4J)2srS;^Xc83icf7N=9Y%7jW)y@7{e9b4A0q=3gvf+H$_kpAaZuPI-oL*@ynXp{0Fgpxrzd)m9i+8z^x!>dYZGd;fNueO5Lb*yzqpKXwU+Y7zf1kXYBoTiD#aU0-Hy?e0c&W z=qIzpo;{~PBq#y!uwR4CiRqAz8~Zh6;BT3Z$?TEOpC1E};LW~4HV*Gq#@~kR3jy7yFk_ zQ^9e6lT|{K$4^Esyvkq>Jc>81cg#^%VGjp~ zA^5$5S7ty)kY$!RhX86|0>bZDUc3OFxC3$O>H)+sF^Syp%y7Us6(fgY)$|Je-&i z0Ln^`83KU+AxA9>Ql^lA;17;WJPVuQZd8X|R(xg}oad zK&FFNZeW}3l1{@DH^%DpDc$I1z@fYVgk=RWlUa5grIa#wOY9y4hZ$@~BaoKvxPx=& zJPID|hz4T<9rJH7C8T5($X-eSux;0mSixr~M4y=$Y>p>l1R_!aU1;3yW=G`fEBWDi zEHvXCz$Z_5(|T1uxyuuyMW4Bc1eXK;PEkpT1FHdW4p|kxMHysgE<6Pa-G3f57UE-; zuQ|lx8+ye21di+!@Z&izil1LNo)2*G=+UD4qp7-;64Qc_!zPo2Ed>!sNS4V5K|n1AHaACIco2uS(HE>qLj$HxHF5YYE%{n>R}&H%j$ zoV|MWMU#E!4(kXB99^DJNaYRSK0 z6X3`C(dlfa5plkS>Z7{#X}o<8`|#yRZ#D@6xv%?DR5p>y_Xi}%Q?&Knn8=;u|NjpV z@)v}a$2dSDeSTUND9GKg{`0Nt0MHzG4y%7uFND(1OyO1g!mH}~tUZf`fH|4E{Lg_LTZ9#ZOC|f2@q0`*I)IGCpR{DcZ~svM_{+9C+r0d zlf$T#XdJMf&2}6(VfqIaHgNX{fPUb#C{qxA)eCv!FO&B7wMAaLO}gjA$Wm>b3H#lq zZ>P?C$z*jzdKc-xmUEx^S9u}6ZeLo5m&j!j$+#sdJ?(Ct@0GG@x>MF;7T)Rw*eGsf z{UnsPviC1#&0}^AaA76ov^g4W}Ecb@%AyQb|lL3?5&f z_w@8QP+Z%6l|Max)BXyh4PXxLSho*az0<|uI%nPd;Y@fHTeC0^ZkCG&-4B~&ogtdqquTcFg`GM<)C;45i<6z*|1T}8~0dsMJi7CySnLJL*BPYH+TiNX&MsNFQ z{Vj_B1uVOlxtcvGm#TN^^oJe(=POa;rLM_00zrbaEvsScTl@4iE`F1VfnDE1u@jP~ zKFlLm`#tl_>x7>s9`KPK-_bW5!E@t!^&x?rN))RXLJ;MCjj68C2xO}11qB?HB^Xvq ztvav(2lMuVXI@2AbRI4BEk5IX93yogoz(?)auE*>=R|%>eKw1E7HC=><8@K<)j>2N z%)+%MABP#D3H1rAL=xvgEYp&cx5H+%Mj;*?Rll*3Kq6c>tXpReTJpn2`ztU$oG|&P z@O4h*06a%4`C|i-(5O9Sc)U8d=QlhaqCj*2zpf?nt|yPzSKz z6YULqrx1NbJhVMzIu&W|ExjfSJlp&P?Qi5Ft!-_IGO_?Wfz)OnY50N~^TiHWggm?; z>hk3bAil!pE-+`sE@(&|aG3@`t5aaxMesIjbeJILWkYu3kX|8#2B}Aw=7Kk7+UFWU z)i9usP{>6Covdj28uck(k^4|^Qa^X$=&&*kXb!wijvmY-aDujo zRv=75rS$!kF?0|Kz=McbQRqa){S(xeUxO#IPEqm8n3x!bd-Jk7+#)7#2+-0}kI@#) zPt^_@GrUTN4iXuUS`02}^Ma;=vfs%V7%Urif*n9I1yI`KhmpPDhJ=*`Amq!Xq`JTa zHKiR4hhlw-G2X=uDyQP%!x~JB;=(BgBJ2bN0_PH`WAKZUf5U;D2)*7uEp&$hRb5p+(RndSEOMeQRzK}}LfMP+4p z1iC<6M+V`Yg^y=tWobIc#m3$MwXbaRdDQ8LRpU6If$J&f0G`Aj(;t3%q$f7EZ_(|p!|eB z^Q{I&<^ghZ+A=6V)2xqf%;l3%^3X0s^=EO(gI(?+lFOHe&JafFG|WFPn@`rkDAce- zsqSw;_nbC(YuSW;TcCmE*%wOzHcy?QW$^#ggF-vz=Hm6*uFb;Brht^vE*o@@8KYKWH*zcmH&9oy=W!-yps&R=@z}eV=T~1i z;0Q|~LmYumR==>LYBh^`%UG!R6Oexb!K#kAi7U5#V~%hmsLzZeK`UP8s0|9gFH0&p zD1_bHw>9EZ7kXc)a?$G?W^aV&y6a67s(qzrgfCGU_Lb4NY@}J=a-_0`p5bVz3CuP` zl2dDUh!dLvPg@5Kh@6BqPh$#t)=q_Q7#)T-#rb22S-Wt6rF!f(oket7nt>yUP#5I#_g2oM-frDbq=gSU=$prROLA#| ze%>Lxupq~`8fNj^btyEj!WP%m#$&3NF^u?;HuDyq`lklzqCWXMomp~g3KoybH^%tKJw zjeV_w_D%#d^&qKF)i=VUAOxNmv}M)WvLCySzA|pVAv)A`(Dsz76_$lTP_a2bvf6>9 zZ##lYjag)`ldYhGE@HBHksPs6T%lxlli3`9Ii|@oIW|-T!laT|eI0D3j1?}cWr-%$ zC}Z9w99uk^5iX@JBw)7X1kObJQ>N2#DL<@+9Zw3xx@22~>bL8`drj7$`RaRo7x za2?c`T_E%h?xqodW+K^sh(`jH1X)6`V2W8XB`emWyOaZ37 zSvfew!f$-~BCwtvJQ@VbXNE^jcn~ZmCLB?ydO1 z1U!2Ri=U46TUXzc_%p_g;P)*H@B(sl1#4l%(k;YYSBo*A+D>%X*I}yaRfI1Yy(Ji= z)`1zD7k;H(*;ZAkJvrAsCg=%T!^?1_VR)ecq(vg(&?+8}Hw>%E5B0;$z?9enDr-B* zIbPrxEIhlP5MZb2-IkMOh+gmHlfd z>iQIlnl+d=%0@{Ckz+)oar#cuTFsnLORk}jP?dQ-iz!>?VUGOFmNc@`z2%sMQrmaKQ>M){+XMOQ%X=vsxW70eh*8l@*d47p z8Pgb4pQM~v{!gI!nfA8-D2yWHAvzx=>zsIPTT4!J7V0D%*zOm};Q?e>pB_-)NDKlL zE;Tk9NtH7b}n{%}FA(=beX6odZ z-$&8E^}3TqN%Q`5P5Mx3{m_!8DHiuTc&Pa|u@gp@G(2qb*D&W1Y_JP!PSu*EjZ6(- zZm!G5u)_0n34YAzm?lApdo4{(O@A5}sv&fTN>AW2f<&kRZ_53GW^ytDMLtWsIO*J# zw@sR@`2yYRn*AA#N8+p77j^a-IDJW$5{$IpzJx!TH1<9OoYPq6FW-`7i@^h{SN`&* zPQeKBBb)MjukN>hQ5!YM1pYT?g2YIeAz!t!05}(9cNR3{4oE>t+yy0@`b9ZcR zZrY=00*!&6S8K@)5lxDhD5=UveE&zD`yXKwncnt=Yd(k<~jV z1mv=z46ez`TuVg_<;f=r#7SGEMsd27Im2h2vRm;5T<}-t{J-DvJ~Y27D=Q~YfWjFM p)_-cLOy`7?j_&_e0Wc=N^pKme{+?_-eA<|p$qv))Sq6Jg{Rdw+tPB7E literal 0 HcmV?d00001 diff --git a/sync/README.md b/sync/README.md index ab7e0ac..abfc2d6 100644 --- a/sync/README.md +++ b/sync/README.md @@ -1,4 +1,4 @@ -# sync +# Sync This directory includes a helper script for synchronizing contents from specified Tekton repositories to this repository. @@ -6,9 +6,13 @@ from specified Tekton repositories to this repository. To run this script locally, set up a Python 3 environment with appropriate Google Cloud Platform credentials, and execute the following command: +**Note:** This is a [link](../DEVELOPMENT.md) for the steps to run the entire website locally. + ```bash +python3 -m venv tkn_web_env +source tkn_web_env/bin/activate pip3 install -r requirements.txt -python3 sync.py +python3 sync/sync.py ``` ## Usage @@ -23,3 +27,71 @@ sync.py: Try --helpfull to get a list of all flags. ``` + +## Configuring Directories + +The config directory should include the configuration for syncing/curating contents from +specific Tekton repositories. + +See `pipelines.yaml` and `triggers.yaml` for more instructions. These two +YAML files control the synchronization/curation from the `tektoncd/pipeline` +and `tektoncd/triggers` repositories respectively. + +The YAML files here are used by the scripts in `../sync`. + +The yaml sync file requires the following schema + +```yaml +# Each YAML file under sync/ configures how helper/helper.py synchronizes +# contents of various versions from its source of truth (usually a GitHub +# repository of a Tekton component, such as tektoncd/pipelines) to +# content/ (for the lastest version) and vault/ (for earlier versions). + +# The name of the component. +# sync.py will use this value to build directories in content/ and vault/. This is used to for the list on the redenred web website. +component: Foobar +# The order of the component. +displayOrder: 0 +# The GitHub repository where documentation resides. +repository: https://github.com/tektoncd/foobar +# The directory in the GitHub repository where contents reside. +docDirectory: docs +# The tags (versions) of contents to sync. +# Note that sync.py and related script reads tags in the order specified in +# the following list; the first entry in tags will automatically become the +# latest version of contents. +tags: + # The name of the tag in the GitHub repository. +- name: master + # The name to display on tekton.dev. + # sync.py will use this value in the version switcher and other places. + displayName: master + # Key-value pairs of files to sync, where the key is the original filename + # and the value is the new filename. + files: + - foo.md : bar.md +# To add a new version, append to the list as below +#- name: v0.8.2 +# displayName: v0.8.x +# files: +# - myfiles.md: myfiles.md +# The link to the GitHub tag page. +archive: https://github.com/tektoncd/foobar/tags +``` + +## Mental Model + +This is a quick diagram that will help you develop a mental model on how the sync works. + +![logical flow of the sync program](../static/images/mental_model.png) + +## Running with Docker + +To build the docker file + +**Note: If you trying running the container without supplying a config directory it will fail. Only copy specific values instead of the entire directory. We're primarily trying to avoid pulling in config/, since it's confusing thatit will not be used.** + +```bash +# You must cd into the correct directory to build the image +docker build -t tekton/web sync/. +``` \ No newline at end of file diff --git a/sync/requirements.txt b/sync/requirements.txt index 3eff7fd..83745ef 100644 --- a/sync/requirements.txt +++ b/sync/requirements.txt @@ -3,4 +3,9 @@ PyYAML==5.1.2 google-cloud-storage==1.23.0 Jinja2==2.11.1 google-auth==1.14.0 -absl-py==0.9.0 \ No newline at end of file +absl-py==0.9.0 +urlopen==1.0.0 +markdown==3.1.1 +lxml==4.5.2 +coverage==5.3 +flake8==3.8.3 \ No newline at end of file diff --git a/sync/runtime.txt b/sync/runtime.txt index 548d713..e69de29 100644 --- a/sync/runtime.txt +++ b/sync/runtime.txt @@ -1 +0,0 @@ -3.7 \ No newline at end of file diff --git a/sync/sync.py b/sync/sync.py index 073e5ae..6dff2d4 100644 --- a/sync/sync.py +++ b/sync/sync.py @@ -7,21 +7,31 @@ import fileinput import os import re import shutil +import markdown +import os.path +import wget +import logging +import yaml +from urllib.request import urlopen +from urllib.request import HTTPError +from urllib.request import URLError +from lxml import etree from absl import app from absl import flags from jinja2 import Environment from jinja2 import FileSystemLoader -import wget -from yaml import load -from yaml import Loader + FLAGS = flags.FLAGS # Flag names are globally defined! So in general, we need to be # careful to pick names that are unlikely to be used by other libraries. # If there is a conflict, we'll get an error at import time. -flags.DEFINE_string('config', os.path.dirname(os.path.abspath(__file__)) + '/config', 'Config directory', short_name='c') +flags.DEFINE_string( + 'config', + os.path.dirname(os.path.abspath(__file__)) + '/config', + 'Config directory', short_name='c') CONTENT_DIR = './content/en/docs' JS_ASSET_DIR = './assets/js' @@ -32,20 +42,101 @@ BUCKET_NAME = 'tekton-website-assets' GCP_NETLIFY_ENV_CRED = os.environ.get('GCP_CREDENTIAL_JSON') GCP_PROJECT = os.environ.get('GCP_PROJECT') -RELATIVE_LINKS_RE = r'\[([^\]]*)\]\((?!.*://|/)([^)]*).md(#[^)]*)?\)' +LINKS_RE = r'\[([^\]]*)\]\((?!.*://|/)([^)]*).md(#[^)]*)?\)' jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) -def transform_links(link_prefix, dest_prefix, files): - for f in files: + +def get_list_of_files(prefix, list_dic): + """"get all the values from the key-value pairs and + inserts them into a list with a prefix""" + files = [] + + for f in list_dic: for k in f: - dest_path = f'{dest_prefix}/{f[k]}' - for line in fileinput.input(dest_path, inplace=1): - line = re.sub(RELATIVE_LINKS_RE, r'[\1](' + link_prefix + r'\2\3)', line.rstrip()) - print(line) + files.append(f'{prefix}/{f[k]}') + + return files + + +def transform_text(link_prefix, dest_prefix, files, url): + """ change every link to point to a valid relative file or absolute url """ + + logging.info(f'Running: transforming files in {dest_prefix}') + payload = (url, link_prefix) + list_of_files = get_list_of_files(dest_prefix, files) + set_lines(list_of_files, payload, transform_links) + logging.info(f'Completed: transformed files in {dest_prefix}') + + +def transform_links(line, url, link_prefix): + line, is_transformed = sanitize_text(link_prefix, line) + links = get_links(line) + if is_transformed: + for link in links: + link = link.get("href") + if not(os.path.isfile(link) or is_url(link) or is_ref(link)): + line = line.replace(link, github_link(url, link)) + print(line) + + +def set_lines(files, payload, callback): + """ get all the text from the files and replace + each line of text with the list lines """ + for line in fileinput.input(files=(files), inplace=1): + # add a line of text to the payload + # Callback function will mutate text and set the lines provided + callback(line, *payload) + + +def github_link(url, link): + """ given a github raw link convert it to the main github link """ + return f'{url.replace("raw", "tree", 1)}/{link}' + + +def sanitize_text(link_prefix, text): + """ santize every line of text to exclude relative + links and to turn markdown file url's to html """ + old_line = text.rstrip() + new_line = re.sub(LINKS_RE, r'[\1](' + link_prefix + r'\2\3)', old_line) + return (new_line, old_line == new_line) + + +def is_url(url): + """ check if it is a valid url """ + try: + urlopen(url).read() + except (HTTPError, URLError): + return True + except ValueError: + return False + + return True + + +def is_ref(url): + """ determine if the url is an a link """ + if len(url) <= 0: + return False + + return url[0] == "#" -def retrieve_files(url_prefix, dest_prefix, files): +def get_links(md): + """ return a list of all the links in a string formatted in markdown """ + md = markdown.markdown(md) + try: + doc = etree.fromstring(md) + return doc.xpath('//a') + except etree.XMLSyntaxError: + pass + + return [] + + +def download_files(url_prefix, dest_prefix, files): + """ download the file and create the + correct folders that are neccessary """ if os.path.isdir(dest_prefix): shutil.rmtree(dest_prefix) os.mkdir(dest_prefix) @@ -53,92 +144,132 @@ def retrieve_files(url_prefix, dest_prefix, files): for k in f: src_url = f'{url_prefix}/{k}' dest_path = f'{dest_prefix}/{f[k]}' - print(f'Downloading file (from {src_url} to {dest_path}).\n') os.makedirs(os.path.dirname(dest_path), exist_ok=True) - wget.download(src_url, out=dest_path) - print('\n') + logging.info(f'Downloading {src_url} to {dest_path}...\n') + try: + wget.download(src_url, out=dest_path) + except (FileExistsError, URLError): + raise Exception(f'download failed for {src_url}') + logging.info('\n') - -def verify_name_format(word): - pass + return True def remove_ending_forward_slash(word): + """ remove the last character if it is backslash """ return word[:-1] if word.endswith('/') else word -def sync(sync_config): - component = sync_config['component'] - repository = remove_ending_forward_slash(sync_config['repository']) - doc_directory = remove_ending_forward_slash(sync_config['docDirectory']) - tags = sync_config['tags'] +def get_file_dirs(entry, index, source_dir, dest_dir): + """ return the files and there directories. Their relative and absolute + counterpart is needed to download the files properly to the website """ + tag = entry['tags'][index] + repository = remove_ending_forward_slash(entry['repository']) + doc_directory = remove_ending_forward_slash(entry['docDirectory']) + host_dir = f'{repository}/raw/{tag["name"]}/{doc_directory}' + files = tag['files'] + + return (host_dir, source_dir, dest_dir, files) + + +def download_resources_to_project(yaml_list): + """ download the files based on a certain spec. + The YAML sync spec can be found in sync/config/README.md """ + for entry in yaml_list: + component = entry['component'] + + for index, tag in enumerate(entry['tags']): + if index == 0: + # first links belongs on the home page + download_dir = f'/docs/{component}/' + site_dir = f'{CONTENT_DIR}/{component}' + else: + # the other links belong in the other versions a.k.a vault + download_dir = f'/vault/{component}-{tag["displayName"]}/' + site_dir = f'{VAULT_DIR}/{component}-{tag["displayName"]}' + + dirs = get_file_dirs(entry, index, download_dir, site_dir) + + host_dir, source_dir, dest_dir, files = dirs + download_files(host_dir, dest_dir, files) + transform_text(source_dir, dest_dir, files, host_dir) + + +def get_files(path, file_type): + """ return a list of all the files with the correct type """ + file_list = [] - # Get the latest version of contents - url_prefix = f'{repository}/raw/{tags[0]["name"]}/{doc_directory}' - dest_prefix = f'{CONTENT_DIR}/{component}' - files = tags[0]['files'] - print(f'Retrieving the latest version ({tags[0]["displayName"]}) of Tekton {component} documentation (from {url_prefix} to {dest_prefix}).\n') - retrieve_files(url_prefix, dest_prefix, files) - transform_links(f'/docs/{component.lower()}/', dest_prefix, files) + # walk through every file in directory and its sub directories + for root, dirs, files in os.walk(path): + for file in files: + # append the file name to the list if is it the correct type + if file.endswith(file_type): + file_list.append(os.path.join(root, file)) - # Get the previous versions of contents - for tag in tags[1:]: - url_prefix = f'{repository}/raw/{tag["name"]}/{doc_directory}' - dest_prefix = f'{VAULT_DIR}/{component}-{tag["displayName"]}' - files = tag['files'] - print(f'Retrieving version {tag["displayName"]} of Tekton {component} documentation (from {url_prefix} to {dest_prefix}).\n') - retrieve_files(url_prefix, dest_prefix, files) - transform_links(f'/vault/{component.lower()}-{tag["displayName"]}/', dest_prefix, files) + return file_list -def get_component_versions(sync_configs): +def yaml_files_to_dic_list(files): + """ return a list of yaml files to a sorted + list based on a field called displayOrder """ + + dic_list = [] + + for file in files: + with open(file, 'r') as text: + # get the paths from the config file + dic_list.append(yaml.load(text, Loader=yaml.FullLoader)) + + dic_list.sort(key=lambda x: x['displayOrder']) + + return dic_list + + +def get_tags(list): + """ return a list of tags with, there name, and displayName """ + tags = [] + for tag in list['tags']: + tags.append({'name': tag['name'], 'displayName': tag['displayName']}) + return tags + + +def get_versions(sync_configs): + """ return the list of all the versions and there tag, name, archive """ component_versions = [] for sync_config in sync_configs: component_versions.append({ 'name': sync_config['component'], - 'tags': [ {'name': tag['name'], 'displayName': tag['displayName']} for tag in sync_config['tags'] ], + 'tags': get_tags(sync_config), 'archive': sync_config['archive'] }) return component_versions -def prepare_version_switcher_script(component_versions): - script_template = jinja_env.get_template('version-switcher.js.template') - script = script_template.render(component_versions_json=json.dumps(component_versions)) - with open(f'{JS_ASSET_DIR}/version-switcher.js', 'w') as f: - f.write(script) - - -def prepare_vault_landing_page(component_versions): - md_template = jinja_env.get_template('_index.md.template') - md = md_template.render(component_versions=component_versions) - with open(f'{VAULT_DIR}/_index.md', 'w') as f: - f.write(md) - - -def scan(dir_path): - entries = os.scandir(dir_path) - sync_config_paths = [] - for entry in entries: - if entry.name.endswith('.yaml'): - sync_config_paths.append(entry.path) - elif entry.is_dir(): - scan(entry.path) - - return sync_config_paths - -def main(argv): - sync_config_paths = scan(f'{FLAGS.config}') - sync_configs = [] - for sync_config_path in sync_config_paths: - with open(sync_config_path) as f: - sync_config = load(f, Loader=Loader) - sync_configs.append(sync_config) - sync(sync_config) - sync_configs.sort(key=lambda x: x['displayOrder']) - component_versions = get_component_versions(sync_configs) - prepare_version_switcher_script(component_versions) - prepare_vault_landing_page(component_versions) +def create_resource(dest_prefix, file, versions): + """ create site resource based on the version and file """ + resource_template = jinja_env.get_template(f'{file}.template') + if ".js" in file: + serialize = json.dumps(versions) + resource = resource_template.render(component_versions_json=serialize) + elif ".md" in file: + resource = resource_template.render(component_versions=versions) + + with open(f'{dest_prefix}/{file}', 'w') as f: + f.write(resource) + + +def sync(argv): + """ fetch all the files and sync it to the website """ + # get the path of the urls needed + config_files = get_files(f'{FLAGS.config}', ".yaml") + config = yaml_files_to_dic_list(config_files) + # download resources + download_resources_to_project(config) + # create version switcher script + create_resource(JS_ASSET_DIR, "version-switcher.js", get_versions(config)) + # create index for valut + create_resource(VAULT_DIR, "_index.md", get_versions(config)) + if __name__ == '__main__': - app.run(main) \ No newline at end of file + app.run(sync) diff --git a/sync/test_sync.py b/sync/test_sync.py new file mode 100644 index 0000000..f36898b --- /dev/null +++ b/sync/test_sync.py @@ -0,0 +1,223 @@ +import unittest +import tempfile +import shutil +import ntpath +import os + +from sync import get_links +from sync import transform_text +from sync import is_url +from sync import is_ref +from sync import remove_ending_forward_slash +from sync import get_tags +from sync import get_file_dirs +from sync import download_files +from sync import yaml_files_to_dic_list +from sync import get_files +from sync import get_list_of_files + + +class TestSync(unittest.TestCase): + + # Utils + + def path_leaf(self, path): + head, tail = ntpath.split(path) + return tail or ntpath.basename(head) + + def read_and_delete_file(self, name): + file = open(name, "r") + text = file.read() + file.close() + os.remove(name) + return text + + # Tests + + def test_get_list_of_files(self): + """ get all the values from a list of dics and return a list """ + expected = ["/prefix/f.tmp", "/prefix/t.xt"] + result = get_list_of_files("/prefix", [{"_": "f.tmp", "__": "t.xt"}]) + self.assertEqual(result, expected) + + def test_multiple_get_links(self): + """ This will ensure that get links will + return a list of multiple md links """ + expected = ["www.link.com", "./link"] + result = get_links("this is a [link](www.link.com) and [link](./link)") + + for index, link in enumerate(result): + self.assertEqual(link.get("href"), expected[index]) + + def test_is_ref(self): + """ Verify if a string is a reference. A reference is + defined as a string where its first character is a hashtag """ + self.assertEqual(is_ref(""), False) + self.assertEqual(is_ref("#footer"), True) + self.assertEqual(is_ref("www.google.com"), False) + + def test_remove_ending_forward_slash(self): + """ Remove a slash if it is the last character in a string """ + actual = remove_ending_forward_slash("www.google.com/") + expected = "www.google.com" + self.assertEqual(actual, expected) + + def test_get_tags(self): + """ map a list of dictionaries to only + have name, displayName feilds """ + expected = [{'name': 'test_tag', 'displayName': 'test_display'}] + tags = {'tags': [ + { + 'name': 'test_tag', + 'displayName': 'test_display', + 'files': [] + }, + ]} + + self.assertEqual(get_tags(tags), expected) + + def test_download_files(self): + """ Download file to tmp directory if url is valid """ + expected = True + dirpath = tempfile.mkdtemp() + actual = download_files( + "https://raw.githubusercontent.com/tektoncd/pipeline/master", + dirpath, + [{"README.md": "README.md"}] + ) + shutil.rmtree(dirpath) + self.assertEqual(actual, expected) + + dirpath = tempfile.mkdtemp() + self.assertRaises( + Exception, + download_files, + "http://fake.c0m", + dirpath, + [{"test": "test"}] + ) + shutil.rmtree(dirpath) + + def test_yaml_files_to_dic_list(self): + """ convert a list of files into a list of dictionaries """ + # create a tmp file with yaml txt + text = "{displayOrder: 1}" + actual = None + tmp_name = None + + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_name = tmp.name + tmp.write(text.strip().encode()) + + expected = [{'displayOrder': 1}] + actual = yaml_files_to_dic_list([tmp_name]) + self.read_and_delete_file(tmp_name) + self.assertEqual(actual, expected) + + def test_get_files(self): + """ create a list of files within a + directory that contain a valid extension""" + expected = None + actual = None + + with tempfile.NamedTemporaryFile(delete=True) as tmp: + expected = [tmp.name] + actual = get_files("/tmp", self.path_leaf(tmp.name)) + + self.assertEqual(actual, expected) + + def test_get_file_dirs(self): + expected = ( + 'https://github.com/tektoncd/cli/raw/master/docs', + "/tmp", + "/tmp", + [{"README.md": "_index.md"}] + ) + + entry = { + "component": "CLI", + "displayOrder": 2, + "repository": "https://github.com/tektoncd/cli", + "docDirectory": "docs", + "tags": [ + { + "name": "master", + "displayName": "master", + "files": [ + { + "README.md": "_index.md" + } + ] + } + ], + "archive": "https://github.com/tektoncd/cli/tags" + } + + actual = get_file_dirs(entry, 0, "/tmp", "/tmp") + self.assertEqual(actual, expected) + + def test_get_links(self): + """ return a list of links formated in markdown in a given string""" + actual = "www.link.com" + + expected = get_links("") + self.assertEqual([], expected) + + expected = get_links("[link](www.link.com) this is a link") + self.assertEqual(actual, expected[0].get("href")) + + def test_is_url(self): + """This will return a test to see if the link is a valid url format""" + expected = is_url("http://www.fake.g00gl3.com") + self.assertEqual(True, expected) + + expected = is_url("http://www.google.com") + self.assertEqual(True, expected) + + expected = is_url("http://www.github.com") + self.assertEqual(True, expected) + + expected = is_url("./sync.py") + self.assertEqual(False, expected) + + expected = is_url("www.github.com") + self.assertEqual(False, expected) + + def test_transform_text(self): + """Ensure that transform links will turns links to + relative github link or existing file name""" + + expected = """ + [invalid-relative-link](test.com/./adw/a/d/awdrelative) + [valid-relative-link](./sync.py) + [valid-absolute-link](www.github.com) + [invalid-absolute-link](https://website-invalid-random321.net) + [valid-ref-link](#footer) + """ + text = """ + [invalid-relative-link](./adw/a/d/awdrelative) + [valid-relative-link](./sync.py) + [valid-absolute-link](www.github.com) + [invalid-absolute-link](https://website-invalid-random321.net) + [valid-ref-link](#footer) + """ + + actual = None + tmp_name = None + + # write to file + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_name = tmp.name + name = self.path_leaf(tmp_name) + tmp.write(text.strip().encode()) + + # mutate file + transform_text("", "/tmp", [{name: name}], "test.com") + # read and delete file + actual = self.read_and_delete_file(tmp_name) + + self.assertEqual(actual.strip(), expected.strip()) + + +if __name__ == '__main__': + unittest.main() -- GitLab