From 68e20ea4114aecd35ac8ab64e1e79c18a39fccb8 Mon Sep 17 00:00:00 2001 From: Superjom Date: Tue, 14 Nov 2017 18:26:48 +0800 Subject: [PATCH] add backend design --- docs/backend_architecture.md | 51 ++++++++++++++++++ docs/images/visualDL-backend-architecture.png | Bin 0 -> 13236 bytes 2 files changed, 51 insertions(+) create mode 100644 docs/backend_architecture.md create mode 100644 docs/images/visualDL-backend-architecture.png diff --git a/docs/backend_architecture.md b/docs/backend_architecture.md new file mode 100644 index 00000000..4ef9a6ea --- /dev/null +++ b/docs/backend_architecture.md @@ -0,0 +1,51 @@ +# 后端架构设计 +总体上划分为3层,由上至下分别为: + +- Service Layer + - `Server` 提供服务 + - `Frontend API` 后端前端的交互接口 +- Logic Layer + - SDK, 负责为其他应用提供使用接口 + - python SDK,方便基于 python 的应用的读取 + - C SDK, 方便应用嵌入 + - Information Maintainer, 负责周期性将数据预读到内存中,方便Server直接读取 +- Storage Layer + - 负责Log 的结构化存储 + - Protobuf API, 底层的存储结构 + +具体架构图如下 + +

+ +

+ +以下展开一些基本的想法 +## Service Layer +### server +- 基于简便的server框架实现,提供两种服务 + - 作为main程序,支持一个前端网页 + - 提供一系列的HTTP接口,以JSON格式支持与前端的交互 +### Frontend API +- `Frontend API` 以单独模块的方式封装了与前端交互的接口逻辑,server依赖 `Frontend API` 与前端交互 + +## Logic Layer +### Information Maintainer (IM) +IM提供具体的数据处理以及计算,比如数据抽样,方差计算(histogram)等, +为了支持类似 embedding 等比较计算复杂的可视化,中间的计算结果以异步的方式更新。 + +IM 与 Server 的关系是 + +- IM 异步更新共享内存,Server 直接读取共享内存 +- IM需要提供接口方便Server读取数据 + +### SDK +- Python SDK 用来支持任何Python界面的应用。 + - 比如 Paddle, Tensorflow 等平台能够很方便地使用 Python SDK 初步支持可视化 +- C SDK 用来方便嵌入到 C/C++ 的应用中 + - 类似Paddle的平台可以通过嵌入 C 的方式实现原生可视化 + +## Storage Layer +### Protobuf API +- 决定了底层的数据格式,protobuf 提供了序列化的公共界面 +- SDK通过Protobuf API操作底层数据 + diff --git a/docs/images/visualDL-backend-architecture.png b/docs/images/visualDL-backend-architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..5d5f158ef7cd0142e8782e260f18a2a37a88f8ac GIT binary patch literal 13236 zcmb80cUV*Fw(fO_A{GPz=`Az~2!=UCW57V66dRaT0w?qHD89sGDvieV8+alxiZIp&6;no^M#TnKC4o20?# zmAJ`yZT_chW`y67eOAaw(@HR`mOj%~m1Nc766p%ZT1=zr=uJW)F-kbR7E)%0%N zrXNVYiG6XN+p|5v2DKaam{o~VDz9$p*ZFhcP}OYT@|wFBi3;V+j2#V9lvV;LbqFg5 zGLcHxP=QQ*hpTgl;@8pT`J24BJTxug*)&G8Tu@+wJR-R8rQwwj|KW;XWZP}2JCt2J zD+hf!8d4vgx!ecy>cP&nf6>V_F>Y$3l*fD$&7XE@bqJDTV+%gw^O~-?+uB&?y{Kgu zmgUpug$utm+9)WqX$0#%%1LW2+#B<<1`m}_8dA=?$_>PtOcwSFX>JQxsYu^7dkk4pW@nz43wTf;fz+li;u zj&MiSs;94Dl18kW>A|P|k*Yag|11G)&5O&D;K}X+EwJ57V-(o9d|$bbtv6&?guevSCszUYGm1TpOE#%2mU57|yCY*s#i5UB>PVYCXLw z8|idcqmSLWDHEv3Am8;WR-9JJ@Q`xmq8#o?x{aGfyrcz-RkT6515S;;w zOnx3(n%5tbcE@JN&Jt7Dl>{~XIN24UFv1;ui6JL4F@4-QDwBfRR4ynPjrpi(xU8a8 zVpzVo1JVtDJ06r~gPP-e-Vba_5zhQk1if!?PE!ap|v>>Q#It_<)&^+Ut!j&U9Ay!)n181 ze_M6LhS@646rnjbt*TBW%ZqLjU6_qNli$ip=^9?T4|ylPDB+Xa#o5t$o}KP#rs`P7 zKqRk~PjY%8D4iQBrk6&@bMc`z2rh{25+aMglZzxPsSzJ|y=@Fb7zm~(isj?&9l+Rc zKIw?A{9pEGIwd!QVq(1lUjz5t7rnUG(4-4e(7Ys*Y(tB|HU0sJ-UcF;<3*RVr%4fk z+n3vuVC%~T8SLW-uZl6k@LYT0S?ms*Y3y=?lvcFARB@;%B>77DGFLR$pRF<-yHk{q9@E(*!_ znVqU)B%>N9kGJgN+%$$ol$xqsDfw0YSq+9n_gJ6T*Cv_)S5z9_{Me6OAiZ+2rNBpw zD~+8_ILR7QHUZ>Hwpf={c%gxjs=QrRn3G<*CApCN;4^9rOWkeL1f_22X7J9 z(eWOxJ6*iZ%tJ^dZv>`D{^1%m5pN8TDv$|8BA)B|kJs%n-V4HGopj z8|J7gmPx>%p_G!I)K48TgShuTTKv2TXL*7;;xbLQl&|BsMINj_xV6u*so3XV9`&!v z4Tb)D?f$ot{*NcR&+(EL1gm9U9KZKr!%GD@)Wl~FU|$q zrLiMs=`&=EVazp7!Yb@iSksz7qm3PU-WX*M8B7ww+b@q2UEEgYQ|WBvWXlI;#U8-@_hWo|td)X( zk;@agqpBeSCKE=myN6BvRD{ipwVUO2%V3>SHS*g(UlI<*=79qazI9@6WJtIW^H82= zd`eqtVaVEMbodpiyl=-n6?cn3tnWqTN6aS59tS{T807FYVEjFwr8>>w9XON34ueji z8u@F|n)yj9{+Hh2xB=la2zauXDbPEOj~+D{zx&|W&??Y=;Av^VhPY-xV6))qRF8YM zy$-#Y%mjPRo#wjP@UOGP2aB@NG9Eonjbc0S zL+CZS8l?!Wv3H<@O?gYdaZSylcbsk0`QTy zOJ(qR&BGp7Vnl26yKn37=iZ3q(?zL~7wPH|c`1lhY~e(C{KNZm>Re<)G<1REL6iK$ zX{`s(w>z5_@3ebYvYlW@zrfseYN9-Lsc+m0oEl@TSIz2CY{M2D12-^p$7{gsUjGI93(T(9N0dMDlM-%=lXnT+XdjyI zLHNzn`O&7Zx_VxOP@Bbb@YeTL=4-kLMyTRkXd#>|Lv8M95+GKnq6^>kSx(*|rRy2a zwiXw~o$ago<9P<@mKq%AaN_Fk+^?ibjO*0otjv7dF@$m2#6v?0dz)0O~&RH2&6_QIS(^Rf=o!W5bJAR?Sj8H!)4O9+&SaV%i|LskYV}4!? z(p1MtNU&_I3>7J>ozDEL2(6aMrId~}8nhofux_*;5B3_+g#qVoTM$JJo1r!2{ zg-bt$W7KJUUe+;(Q6VKUm+x`+1^}Crvycl-y18j4Skbs$4aTBxP*kxs+=wsb6{eNK zsB}JWbdHX5ehd|(S4s!oK0o%x;SUbfk+yj*Rx_7dg2F0AIrP07_8gB^UiGfT(t{JR z`9v3TxLO1xL{|02agixo8-fkzj}{&e&imlM5N)>o#D(qq;5?YE+|Ezza$4=a}m<7f3b#}-wKkq$Bju!@t-Yq-qb3UI7^aNs^u7=6xngI&=f zV1-tG9}k;N#+$H5`MC&(Ec2$y-Q6rM?C|8VIRw#D4Ilitgp#m6%;mGpyBfl12bBEY z;a@xNGJ)N&J{{`VTuDCBN2vvP92Pf55wvskmxB(QcjyU>4%wEn)hBu+iF(q_lf&P# zx$O$AfxZd zR@S>8CE^5nHsl6wqSPrFd?vhg_}R#e#ItWc0;FLv@iAPkpOyGA&7dF5HrkZy!W5oR z3o=nQRC&AJfI^P>!#xrRDSe{nJ*>8&92N$%5&kf`-wwv^;;LYxrc{< zSiH(yU4Ym$!nlCD=pFUNh&iSyQ^^oF{?yfUXQF&#A?+E8O+YSnFWDVM()8FlP1KcX zGl)?$_E#Sb^y1L;ku$xs+Y)L#a!E*sk@}o$lXCL?N?45q-8I6|bX8~al+9hIC}Ac; z7^SL-jShlA2>7$^X0Tozd(M;w$}$(ANL%aPAEVJ@?nn`(N&p9-!B z+Sfee7vwHfik5)-4t|R29WY85ST;)?j+qX@a1Z2?d?RH7g;q5(Z%>}?LYtHiKdF*x zOG277o?ncz!`)6%<)+I~irtr1^e#HHFP~)soc2c^UE>OGSIH8_w25+JRZIygu$X>f zKQN~1ZsSDpY!-gyftR~*1um@>D1SPgW75Jc=?<^U88f{z3rE~nWtt~#HGp2$E-3fpuwu_7u(?zBiW zQXk{0ck!axpwqebM4xjD+OQ*DGw$!iU@bMi2By!~P>tSDGpIf+t}dEXjDJ0dr_`!e z0~h$r2xWQLrCl7ze}oqa;?DQ2?*#B?KM#8EAro-dtVc#q z;Lp{Q3Q`8N{M?mCXtEg?Jhr|~30R*R=QKQ2z-?`nThn!ByZTHXY zbWXUgScLSl;(A@147MWZFId1pf6O}awoA{CUPFNI}p%qdCh4#$!?<6?XmOZ-X!rBsz^&M zY(0$Gb?GLgDzn83dz1It$USas>ZEVMOLE!fB@>i7>H4Zilc-cwMc}ddtn869FHL%s zfJ<}9OoZ@CSsA@4WsVP^QtuwBfvr2*;!g82bGFNXGeklk3-pi`-|d)@;Imo^HMcrK zcDw0&osWhcyp8*r57!SGLrP?f#gfd}MiX2Q0s2+a*V!Ox1#5I6)HKW@f$Q|@E6k_H zxdwO_XT+De%W|S2dX~ylrvl|xpY5cNTjNQggn)L`!pqx=r5yqq-d~2aufxjgc(0KEI`Bcd{K$nO5(E^|!-A zIck~&U6_pB2w%^NMm{PY^xEnj43p2GZQEJ-2Pp+teoE5GFbq~P9?hN{2YjHlDvK1B z#Hy@vNp**h7pxn2sB_%zXGg;g%0+TN#i}vg)9uF3eKxvGU(3y#?h0}G-XpV=9_~c0 zm{(7rhhrimb*UaN({u38Iap)9)4^qz#u(o%eruu9?h*-&i3TOw28U*A$kNetg46xW z#7vY)USA{Q!8N&>To)FtEKe!J;_Ap$ZOy#TC^Q3o8&Ykv2^PZBnn3F(NG1*Viu;}n z)4BCwp^9wO6k(1%zF1s=0W% z2Ea(b*4EaO$y^vtI1}-wn^zxrQ??$hUlmNEf9)ZGDvm~dZ&k#k*+^|-RPq$|>*L3S zIiGwyOYeO7p}uAN(+8W4o@L0$Suqs){-XY88c6=3$`?f_(h2cq4&K9+=h%_Jj8YRy zzUudG;Zj19POrDGCE#tpoP*?3z!QJOtU4--3cK^q*y4rGAh^H4U{H2QG}Pa}Al9CC z5SAV=!VU?0ikErx$F3PYmFdrk!;6(!7Q=|5*Sq+GH_0pXY2`IfEyKU{m z-W`$q8X4vFXynD*H2U~bodR{&W9!B(?0xnk*&LwFOcJ9u5XF4_9IDYso_AX|CYp*i za?+EDEJej++>2g8R;}n!kl6Np=N|EMki79T`ijq;d?pHRj!U>3>>s4b(YyJVPHE=@ z*RB5Gc@BRm39fWjH`VB_FqC@xJ<2e(SA$t1AgYvm?rDXbWyspew=KQic}pm_uINC@ zrQGh9lXRueXRF`1a$ZbfJZvuu%tIn=8e}Vy?}&eIQBn7brRzk_;>(^lW-L~r7D^vY zDnr!Sl?=i!Y3FOnD1wR$KldGQUiJP~tJ3YK2NO|KPZ}E}rdeo?B^Jc#Hr{prFs`=4 zx$;4qduHDlhUblUee^O88BJlyeaarBRlYPF~3Jue$#h=euDhTyz?%t$0R--@Q@uUi18wzxHxKihz|E}L(aN&&a9d^K_=tCQj6cXp4YDl>*(F3rfYcWsmI zXd9$pooUYL#=z+2U+_@_$knYHs-R8oGnk6>x!)wmt;L@Ud(7GI<=So=s}5PIMR&PT zxf4?_E~GFU(_fw!7RouR;UNt3AQ{W3WLE>Z0x7bV?Np)qsynXPG7rB)21dk}P$XhN ze>aD*-#6VDVD6E;lzkN7OGLw<%c87O!r0e|M2R3a$WDT@Jiw>kZQ~k^`UXY zg13H#gy$3D9ip5cj>BqgH&C0W#<;Bw1fkbvq)PBNWl9b&8w7tMZoe5=+}ZVR-runM?sj+O`?m4ycDp#!Fb(l0mXg|F z=A2}em{%e9`^v%GxMChl0#=oZqMz8GX4acp>4Z6x&arr&75)PryIM!&R+_~%#HuA1YnE%Fv%GdRAOwzs z+?H=c%agG#U7qs4_g4~MmRbG8qC?K)(ze|Gp3YKeuV2&>VhWMeJN;g^$al&gg&+o* zz*s@N;M0b~76ayf3AoU+i)#cKC?^vrYj0>~G#d5kJYBMs5VsI?NQvji^XRBM&GBZG z^!^r^GlKXTncf)3TSGZ2mtpk80v}2u)c1n{zB5QRFQi5WNrknY^3dclMqmQ4?xZPuJnrim{7*(ZyI6YpcE?=BaNv+wRS? z!!-2@<$N==)W#M%7VJ=`Wuij!Gz_wxi{2c&On;v4g~p&7%n*v@-0U813;S3dI<*+~ z5+&5^NN}~_kadvkYxDHAsH?ha@!rViEBvGy4hVfHqt@3zKPU2OOwkQ^PsTO37aYlf zqL7UcBahgJZZLO1`p zL>a34RbBwfJjw{~@z9B>X-q?J&RH1h&fMq0ZuN4F&pXhkbHKRe|dYQmr@rKyEg@Xm#uZC^=o zu)?6Arqc0hJMB{5_s@lG4`*Tz?!yKhB4-Wu31l8%-s>M?V*SP{7p_5Ak=rW$Z@}ao z2T}uWa=<@(ls*|bI*WLdilw0VBqlY+4!^WNOD|H7s*B0$&*|l`mY{o47&fQQB%a`b z`spT7S%IT%ZZf+TLXd+Op{IJEF7Rlj&ho2bjiPj?=4 z$>fg$P|?5ul7EbJgg<%c_Ik4~RqofIwG_7QLDB85ztnNLe12~^604n<`v!`=xb(r2 zy+7{w3JR~O5{BU#IZ?fb_hz-~(XrEg-cPRJgJO@rv){|xXWO|u?vOrjv=n2%dQzIY z#DC!EW;xg%(_)LjTS2_g=y+xD6(J@ebg>)f}oXimNFgQ&(XY@{s&K+B~+jXHx3^59ev3M+SCEz z=AykooQyLx?|5(Kd=A*pxGQe2rq0={z-K=K3Ye35COa^HyULaO@F|H{k>&p9sY{SD`S!#y7CyCsR_3;iLvR?v|b|8PQEo6Ku3*St2!DWD@6 zMiLeL*eov>2jt>``RacSoSyWAM&h`1WQR}wBPaA^`q7bkL3Jy-OW03QS$baEQuPQb zDf9DxfqWyY#6-VK*zad|Tct%^=~j3`t&6RoU`m256uLBMPuK zl(kaHxGheYhtn-~kNCDU68&ocrOqAFs3kQOfQ zC5c^zU$X~cRo1!`p+%t$GzLw)TyfRzkg%7r@-9vK06|j!b?>&?GypYI`>IsBm$H$O zYLSS383+4(pi|u!By|V)^H5K~{51UFd=R$hz`R@ljtMwR z_-v?&BNI#tyzz!h`h{U2(9#j^^=DhUCf6Pi8XcrOFDu;Koh0h^!l#Z`VM;U*IvSrP zJh#lIjU>fhb;V0yWbR4THmXeB2Jk~~nX5rB@6!=5VVXS%d~8b00Yo`C%1@RLXg1Xu zn?D<5sD;m&e-$ISI@|}=M&ov9Im+UCP!e7yFs2>)rK37J1n|Fig@M2CwvMV8-{>33 zIr zImF&e+RQ(*;y<_@C{KSLY=`RQ#;)6;CDH6ug;Jcdk&IM#yZwK7U? zt(_kK&sf!eO2z(_!%9i%X^oNlwQ$M>!cVdUw=bmLr%S3BqNASNzXedeKb!u02K|o# z{l7nvXGn~e7hf( zaS%!Dx#W8i=AprlTfb^vGd#EdF{buwxBw8gLKO|XHoKneJe%(NCC0oKVeZU=DK38X z2f2CZ?^^(vKxi7#LCqO7yEF-w7Y3w2a;gjBMSY_U@3$KCL=$qH^tMOsicrH$PXqEY zDWH`}pFOc%;?9K40$7y@iYw&Z)KDXeFn~5T?=MQ~znN0j^j0%?KCHP_m;}?@KB>kD zbL!MANrV*N${90Dr?;_&*OTbjM*UKYVbejc{1y;H5w3~ibzMI9ynyl39!dKo0n#*k z=grmqqtyN^lh|q3h6++(%A9w4xP0)cJ1eMshhcPK>Z01mwHfpaUpl1bMZ1lUVN&?9ZtN}_XNz5pl!uz06D8i|JOr5L_W8)!e35G zf)QgT$%5z{1Uo73Mkm=0wM`I8-L8xQ0)K)LRi!Q0X+$V~Rt~^)JemAthS&$~1Vv*4 zkymUaN7I7&pIDjR=;fA~G69h~nhU`AS3FcBd^_72WcP#XkK^a5`RbAqwa7b-Yvfeevv$*TC*vcp@K zGk2YPs$#_Qr$qaDsqAN#O(!2luKT}~5)P`ClD$_;NXQ>o*8}C9L^PurS5w>+z z(@ul(@^=$)=Ey6AITj41p0d82bNHR-$Gtcx_p(wh~FEyuU=fphE z1*#M49;h0~EmX;aKjOWw2YZpSm4Tj90rMr+B((i>9TV3Oa7<59U-KO&p>av~ZmtX< z39v{r?RdXqT?~b?x0{K<&vHE9axp|kB6+TVln+F>j(fjdgL?I`y;a}MP!mm0l7g9k zwY9N8imm*02Jw=Xi0yt9>j5=K3eZ-gR}`YhNl?29rO{Ikew@`M6f_j4k#i>q-)IVzMu;W%DQsP&5d=4LXh6*qpfOuIrXo98a@95SukIOmSKlq`LuB1L2 zCd(A#P(Qe(E^}k9xh*%x&`I?Zjrv1ddLp5eVxPKMoQD0d<4Jnvr{P~4h%bS?8SZW- zLa`2Ftc*7Zvz;mSF4+T1tFj4W{iy^eRt$RP43!oHleATpy71$hB@eqGM|l?~tCOp< zE)4IM=d(T~WI7TGQWZzn=GeQZulHvZVbFP6udf+Ts7Cf^YZo@oPhKEc>TPQKRk-NE z@Mopo$PN%Fo5ga0XUf77*pPk#MdaNu9q?H^O5FZ1h3N=-MjC;yrb!h=M7 ze?}PX_qCiHu{Hw+U;dofK^Fw~X32z3-tUn~mFqwwE~IJSgqkt5spe2V>Q+V#y|tB} z2|s`EdiB@IOHRy?jS{-)4pwv7`jvug&xAWKIHpD(rzW?48XI)aJgXZT2-IS zfoA{-06I^8U(5a^8Zb25EZss1LXqb1LY=7_pXRrV(3JiF0V#Am8@K*sj6q_fx*NB) zb6h_#6uE4=`hc2{6n9!(le0U~{5rpYEWZ2mYbDJOVoUFtJ0#ydO(eEQiT>Tra0QgGhPis)i ze8~gnH~*=$dv>4ww8=#_XBZe3kG1?-6#>f0J42Xk`ul?fXv?t^Il*v+e)P=7DJd#o z_eOi}wS${aXUqc1G4k|u> z-$}rBgb&qnmCK6YDV+7Uky-p)Oiav(WoGULnR5xlQwW zmH;9L6`j!1_`T~=HRAc)ZSu}Puv>E@?19f&9QW1stz*?3Apwuig=_jq&bTX%9pc5X z{j6#Wm|jt>?3-ryU^2#Dm+F(i`ePof+L%O=-v})%qldpQJqjp@{8*R7Nnj@~b!Qb9 z`*krxbjBzibMs;nR>6-%AGQpJ+4XH+}F zb$Wf;9l}|9l;S5Bn+a<;X`E)VU>V0iH&x$^R(W-O*SKtG!d$H5YWt3IHQf@4SbOzxCQc((BixXQv)HA`MSm{d zxrfXdcQG^)SK#u0{~O*Pkf^#S)sGpWUY?w4G7nVu23bSl^~HP>#*5KzMK(&L)?A{Pc8VVoOicA_;v`ACGS8g&rLfKzJ()|BYl^De$R^1N^mfyu5Y$JMU&~D9Yq$ctJ17 z-P^1EN&Odm*F0pA1@ZyIBhb?oIsEZXfRG0Cf({{Op0+biK3{VFc@)aBb>AQ-;Ng25 z8EGsldHPXmc1b`bMjFE)v1$rP#1(eP8{3WoA%Qzi->tS%X)c5TF}D97ZTjzo1A+!< w$X=hWUt1%6iX~*%hduBg9T{SG^Ylzu9rW&Ssi_|DX`D0a%G%I!#Rt#-4;OTyIRF3v literal 0 HcmV?d00001 -- GitLab