From 4c45ff4b0290e70c3dc5c470c483d6e184eb19bb Mon Sep 17 00:00:00 2001 From: Maria Kleiner <mmandlis@chromium.org> Date: Mon, 30 Jan 2023 23:58:10 -0800 Subject: [PATCH] add extensions support (#247) --- extensions/chrome_ext_v3/Test.js | 2 + extensions/chrome_ext_v3/app/ExtensionApp.js | 78 ++++++++++++ extensions/chrome_ext_v3/app/arcs.js | 77 ++++++++++++ extensions/chrome_ext_v3/app/config.js | 9 ++ extensions/chrome_ext_v3/app/logo_32x32.png | Bin 0 -> 2519 bytes .../chrome_ext_v3/assets/logo_32x32.png | Bin 0 -> 3127 bytes extensions/chrome_ext_v3/background.js | 35 ++++++ extensions/chrome_ext_v3/content-script.js | 0 extensions/chrome_ext_v3/hello.html | 23 ++++ extensions/chrome_ext_v3/hello.js | 13 ++ extensions/chrome_ext_v3/jello.html | 24 ++++ extensions/chrome_ext_v3/jello.js | 13 ++ extensions/chrome_ext_v3/link-deploy.sh | 2 + extensions/chrome_ext_v3/manifest.json | 35 ++++++ extensions/video/app/Camera.js | 34 +++++ extensions/video/app/CameraNode.js | 54 ++++++++ extensions/video/app/ExtApp.js | 25 ++++ extensions/video/app/ExtRecipe.js | 116 ++++++++++++++++++ extensions/video/app/Resources.js | 29 +++++ extensions/video/app/arcs.js | 64 ++++++++++ extensions/video/app/boot.js | 9 ++ extensions/video/app/config.js | 29 +++++ extensions/video/app/index.html | 11 ++ extensions/video/app/index.js | 11 ++ extensions/video/assets/giantInLake.js | 34 +++++ extensions/video/assets/logo_32x32.png | Bin 0 -> 3127 bytes extensions/video/background.html | 10 ++ extensions/video/manifest.json | 40 ++++++ extensions/video/options.html | 15 +++ extensions/video/plugin/background.js | 11 ++ .../video/plugin/cameras/arcs-camera.js | 91 ++++++++++++++ .../video/plugin/cameras/basic-camera.js | 63 ++++++++++ .../video/plugin/cameras/x-arcs-camera.js | 47 +++++++ extensions/video/plugin/content.js | 31 +++++ extensions/video/plugin/options.js | 18 +++ extensions/video/plugin/virtual-camera.js | 68 ++++++++++ extensions/video/plugin/virtual-stream.js | 97 +++++++++++++++ extensions/video/plugin/web-rtc.js | 95 ++++++++++++++ 38 files changed, 1313 insertions(+) create mode 100644 extensions/chrome_ext_v3/Test.js create mode 100644 extensions/chrome_ext_v3/app/ExtensionApp.js create mode 100644 extensions/chrome_ext_v3/app/arcs.js create mode 100644 extensions/chrome_ext_v3/app/config.js create mode 100644 extensions/chrome_ext_v3/app/logo_32x32.png create mode 100644 extensions/chrome_ext_v3/assets/logo_32x32.png create mode 100644 extensions/chrome_ext_v3/background.js create mode 100644 extensions/chrome_ext_v3/content-script.js create mode 100644 extensions/chrome_ext_v3/hello.html create mode 100644 extensions/chrome_ext_v3/hello.js create mode 100644 extensions/chrome_ext_v3/jello.html create mode 100644 extensions/chrome_ext_v3/jello.js create mode 100755 extensions/chrome_ext_v3/link-deploy.sh create mode 100644 extensions/chrome_ext_v3/manifest.json create mode 100644 extensions/video/app/Camera.js create mode 100644 extensions/video/app/CameraNode.js create mode 100644 extensions/video/app/ExtApp.js create mode 100644 extensions/video/app/ExtRecipe.js create mode 100644 extensions/video/app/Resources.js create mode 100644 extensions/video/app/arcs.js create mode 100644 extensions/video/app/boot.js create mode 100644 extensions/video/app/config.js create mode 100644 extensions/video/app/index.html create mode 100644 extensions/video/app/index.js create mode 100644 extensions/video/assets/giantInLake.js create mode 100644 extensions/video/assets/logo_32x32.png create mode 100644 extensions/video/background.html create mode 100644 extensions/video/manifest.json create mode 100644 extensions/video/options.html create mode 100644 extensions/video/plugin/background.js create mode 100644 extensions/video/plugin/cameras/arcs-camera.js create mode 100644 extensions/video/plugin/cameras/basic-camera.js create mode 100644 extensions/video/plugin/cameras/x-arcs-camera.js create mode 100644 extensions/video/plugin/content.js create mode 100644 extensions/video/plugin/options.js create mode 100644 extensions/video/plugin/virtual-camera.js create mode 100644 extensions/video/plugin/virtual-stream.js create mode 100644 extensions/video/plugin/web-rtc.js diff --git a/extensions/chrome_ext_v3/Test.js b/extensions/chrome_ext_v3/Test.js new file mode 100644 index 00000000..9146102c --- /dev/null +++ b/extensions/chrome_ext_v3/Test.js @@ -0,0 +1,2 @@ +export const Particle = ({ +}); \ No newline at end of file diff --git a/extensions/chrome_ext_v3/app/ExtensionApp.js b/extensions/chrome_ext_v3/app/ExtensionApp.js new file mode 100644 index 00000000..785509a0 --- /dev/null +++ b/extensions/chrome_ext_v3/app/ExtensionApp.js @@ -0,0 +1,78 @@ +/** + * @license + * Copyright 2022 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ +import {App} from '../deploy/Library/App/TopLevel/App.js'; +import {logFactory} from '../deploy/Library/Core/utils.min.js'; +import {XenComposer} from '../deploy/Library/Dom/Surfaces/Default/XenComposer.js'; +import {HistoryService} from '../deploy/Library/App/HistoryService.js'; +import {FirebaseStoragePersistor} from '../deploy/Library/Firebase/FirebaseStoragePersistor.js'; +import {DeviceUxRecipe} from '../deploy/Library/Media/DeviceUxRecipe.js'; +import {CameraNode} from '../deploy/Library/GraphsNodes/CameraNode.js'; + +const log = logFactory(true, 'ExtensionApp', 'navy'); + +const ExtensionRecipe = { + $stores: { + html: { + $tags: ['persisted'], + $type: 'MultilineString', + $value: ` +<div style="padding: 24px;"> + <h3>Hello World ${Math.random()}</h3> +</div> + `.trim(), + } + }, + echo: { + $kind: '$library/Echo.js', + $inputs: ['html'] + } +}; + +export const ExtensionApp = class extends App { + constructor(path, root, options) { + super(path, root, options); + this.services = { + HistoryService + }; + this.persistor = new FirebaseStoragePersistor('user'); + this.userAssembly = [CameraNode, ExtensionRecipe]; + this.composer = new XenComposer(document.body, true); + this.composer.onevent = (p, e) => this.handle(p, e); + this.arcs.render = p => this.render(p); + log('Extension lives!'); + } + async spinup() { + await super.spinup(); + setTimeout(() => { + this.arcs.set('user', 'mediaDevices', { + videoinput: { + deviceId: '545b0c354475465dd731e6fe7414319c2d88f4660c6c108ca43528191638406b', + kind: 'videoinput', + label: 'HD Pro Webcam C920 (046d:082d)', + groupId: '6b5db3734e5bd10ecda9de583b7ef3761be03ce1ab1b49c1a91a312fb91f6de2' + } + }); + }, 1000); +// this.arcs.set('user', 'html', ` +// <div style="padding: 24px;"> +// <h3>Hello World ${Math.random()}</h3> +// </div> +// `); + } + render(packet) { + log('render', packet); + this.composer.render(packet); + } + handle(pid, eventlet) { + // TODO(sjmiles): the composer doesn't know from Arcs or Users, so the PID is all we have + // we should make the PID into an USERID:ARCID:PARTICLEID ... UAPID? UAP? E[vent]ID? + const arc = Object.values(this.arcs.user.arcs).find(({hosts}) => hosts[pid]); + arc?.onevent(pid, eventlet); + } +}; diff --git a/extensions/chrome_ext_v3/app/arcs.js b/extensions/chrome_ext_v3/app/arcs.js new file mode 100644 index 00000000..14f67501 --- /dev/null +++ b/extensions/chrome_ext_v3/app/arcs.js @@ -0,0 +1,77 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ + +// use Library path from configuration + +const Library = globalThis.config.arcsPath; + +// import modules from the ArcsJs Library +// the 'load' function imports modules in parallel + +const load = async paths => (await Promise.all(paths.map(p => import(`${Library}/${p}`)))).reduce((e, m) =>({...e, ...m}),{}); + +export const { + Paths, logFactory, App, Resources, Params, + deepQuerySelector, Xen, + quickStart, + LocalStoragePersistor, + // HistoryService, + // NodeCatalogRecipe, + // MediaService, + // MediapipeNodes, + // FaceMeshService, SelfieSegmentationService, + // ThreejsService, ShaderService, + // TensorFlowService, CocoSsdService, + // LobbyService, + // GoogleApisService, + // // must be last + // ...etc +} = await load([ + 'Core/utils.js', + 'Isolation/vanilla.js', + 'App/TopLevel/App.js', + 'App/Resources.js', + 'App/Params.js', + 'App/boot.js', + 'App/common-dom.js', + //'App/HistoryService.js', + 'Dom/Xen/xen-async.js', + 'Dom/dom.js', + 'Dom/multi-select.js', + 'Dom/code-mirror/code-mirror.js', + 'LocalStorage/LocalStoragePersistor.js', + // 'Designer/designer-layout.js', + // 'NodeGraph/dom/node-graph.js', + // 'NodeTypeCatalog/draggable-item.js', + // 'NodeTypeCatalog/NodeCatalogRecipe.js', + // 'Rtc/LobbyService.js', + // 'Goog/GoogleApisService.js', + // 'NewMedia/CameraNode.js', + // 'NewMedia/MediaService.js', + // 'Mediapipe/FaceMeshService.js', + // 'Mediapipe/SelfieSegmentationService.js', + // 'Mediapipe/MediapipeNodes.js', + // 'Threejs/ThreejsService.js', + // 'Shader/ShaderService.js', + // 'Shader/ShaderNodes.js', + // 'TensorFlow/TensorFlow.js', + // 'TensorFlow/TensorFlowService.js', + // 'TensorFlow/CocoSsdService.js', + // 'TensorFlow/CocoSsdNode.js', + // 'Display/DisplayNodes.js' +]); + +// // memoize important paths + +// const url = Paths.getAbsoluteHereUrl(import.meta, 1); + +// // important paths +// Paths.add({ +// $app: `${url}/../deploy/nodegraph`, +// $config: `${url}/config.js`, +// $library:`${url}/../deploy/Library` +// }); \ No newline at end of file diff --git a/extensions/chrome_ext_v3/app/config.js b/extensions/chrome_ext_v3/app/config.js new file mode 100644 index 00000000..b4669dcc --- /dev/null +++ b/extensions/chrome_ext_v3/app/config.js @@ -0,0 +1,9 @@ +globalThis.config = { + arcsPath: `../deploy/Library`, + logFlags: { + particles: true, + //recipe: true, + //runtime: true, + storage: true + } +}; \ No newline at end of file diff --git a/extensions/chrome_ext_v3/app/logo_32x32.png b/extensions/chrome_ext_v3/app/logo_32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..fa50d9881c484c08ab1f2eaf71b31291055c3350 GIT binary patch literal 2519 zcmb_e4OA568ea0Kn<Sa!g^Y3>BY$V-XJ>zAjtc>60k+6du7Z{_J2SgG*WI0AXJ8jJ z4}k`B@pw93(Gt;aUCf)NC!)Cciz0TlE5xbvUIJ5ZnRVnRLQ_%uW?44tP}kGBdv<od znR&kFeV+Gy=lixIGvk@@<0g;OXf)%~QmqTY8B!1J81O3>6@3p*_e-fcvPKgbtsW7Y z*Q#SRnlU?stZXIQ_AJec9v#Dpc3v0oNC2(T%uNhP47-w7AUp3AycW1;|6v#sI19YQ zV8d)u3hxq9i+uc|qKquIXeCQ=aAE>9H$Vdd53ev#z~lDHbie`!d1)|KZ=*02geWU5 zaFQAj%C==fDWZ>uOgaK#F${xDln%EW4UCDhlXD;fBM1~TqPQL*2pYq|7Ycn~5YWdt z=!Mp&L*c-c1$HTlM5CzR@7MYDI??Auaf+f)j6ewj0SH7c^eRjM@ygSN8LYg_`UFW4 zL@%T=GIlXvvA{s=kPD9#ruE99Oo5!C0Y*Y`9j3Yp0y#E}lk$D;U~rB_c{lIjy^0L5 zco-|WL`9Tc;;&GLAO9=>$ga&6jxnMZk0%^LR^}A|8=-`Zh?cVoB_3VK%VNHd<>wUu znbTA^5}o4X8AbGEiK2VBR++<+AwsV+Kr?I%D|l5CGw+(hTN#D7z-rwP976~^3pdgP zPLl+JnP?0fg4#q*a1@RRHPNK$XHd{L9HTJ*6U?!+L-ctVU|H}mP9BxKP8bR&l1>rb zq7MWH>hwVsHXEJhl@-Rz@@ZBJ3|#93fur?C+(=Re2heUeBBY)p5VM0ZBM!3(<4C=c z#5iJjzEx!N)!M4_hdP}TS-|mARFlJwvpjA_>?Q|?@FYnhb~9;4Forea29`3JNphHN zflmN08{@v4RaM0SM$E|Jc7xe~aE!x&kc=7dFbs#7?MA)9z!DtK@@hW^TZB#(WKg`q z&>OOd&kK#*0u<~Dnqk#$x4^7gW}bsXw}lb%aZiy$^ZhO!kp4#ThoZ}(L-8{{KFJAk zcMt7_{+fK5Dfn&iCPGhPM&5uh49g*;gQXBgPq7F|U`8-tSju1+Hb0_xk~V}p_xFhp zrG|AeUMCM$P!#_CcJ8e=hjfl;=kB-b|7izR*Uh0mLx0(V!Dn}^v7ng2TVUZ<hv7{F zT!gnB-U~A21DnbQdJfpp?(0mmCS_%>JANk7wPeY|NBR~l-^xvh8+Wvo+?Y8dIeva$ zEBBIaoE_QL{QT>>%WtipD1FjYkY_OGW0T(NZ2#ALk4%QkOU>nPMl>arPZ(8y%yP!x zKX_}+A?upHBNsdUU6xC$x?Z@nZB<v>z(INGN$rz)_ua(*Fm`uy-vmFl;f$v`s$RRJ zRH$z0u4+2ejC_<^f3ErOtp`7rdSfq5?v2TAnUXiBOn0h#UtwIr)2osauDss!g)wSU z<Mb`Z<Eo0z+P-+=bncXb^V3(CiupO~y|X3FpGOZ;r!UTbu6tC>HQkBN`<dCRkqd8M z8{O9QeC3IOTiUN<+}}r-_-XE_+^NdNhM3BCA}Z5Q)to<^o^#Gr(fFY*t~vAU<KtIQ z<sC0q&luIPWBJva?W>+U2M<Qh-BfeqMBARSy$2?imDJBHj;gf}#_fFn{yWjzj}N?Z zp{wyPYpLnACD)C`f^gZ@cjanv>)VmhgJ-tXO+GYwB|R2{Cq2@>vw+>&+xqAYhvw+D zjj`KXQsd*7>Ijnu>Db!4zvtD%Q$L;9RK5Gn|I~c>!GmkJ|A{dzsa%lN-jY1)Z}X>U z#kvS&*Qu+w)3=#h3p6$HbuA|~iwm^N-)*^Z<)zY+&EM6|dgI?o0mrRnW3VpvJE&%E z-Qroh*2A0HDjq9d@Ybn;W9i?Vf44dgJMhnjMWg@A&p4eMDDFL<tNm*AqzrMLZ;CET z8|5sWx@B*_(_C5~yCQbNfoTI}C5DuPkL|s%v@)@xzo%m@+H$=odGr1~mb@2`EnDfo z&i-L%!<zW|KQ7ypw(Hw}EIs1<wDYs%edMU@6L|i%+ZiX<*=$^7&aoAN==b}7Na(g~ zq~G|Ub8+3ndoKhYYGy9)yf}7dX~#+F?3}U<4<y!p?8)zltm><8toXcid!Kgmfn~eX zUoEFL9DVX||GM7mJA}5Lb5hm~xb~A<_Ntin=uJ02Y-l{n=WLwV{qp0s?q}{$p6I8l eUL35s^WbgQNBa_TUfHAm^-7zUVSOh#x8z5-A)xaB literal 0 HcmV?d00001 diff --git a/extensions/chrome_ext_v3/assets/logo_32x32.png b/extensions/chrome_ext_v3/assets/logo_32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..13a239202934879d8b432d4c639cdced3544ddb2 GIT binary patch literal 3127 zcmb_e3se(V8XmCtKtV;0XvH=}6cr|q$%7=k<S9Xdihxi>oXkuhkXJK-@X{h8$V0JE zt@uPi++_iEIfy`k-J(#`stAff5o>)c%PLZ+h*a4dK-TV|T~E*EWbWL#_q+f9{r~^n zJCnZ7Z?%c>$Hou@nFzf-{K1`~zot(GEg2s_0B$o>-XSCe&77yd44^|t7C_L{d}%<i zCRpSvz?5<tiYvtgEmp1qXb5t2iB+Lk6rrJriEydHiP~FJPo+q4C+bFyh#^wB6A@DH zcr~#h-Y)=)kHYvk)y0|O7%Kn<<b(#L#L8s~QV{Dz9knX}ef=<<N*RS{qMWF%dWV!? z(K?E|QcY0oX$Xul7z~O%pT-n(IjB8f%(kN-3<RMwxO659Mi2pm3I3Gv9~C%M;}U_t zhu63}@Z>~|&}dWwI$f*P(zGm^QXNib^7(u^1EC`b3=l9Gr_i9Wu!3Cj&VmO)Vrr>M zBULIWdW)!78Le@m0<vRK$W;?&736rPK+fo~sEW>{G4xT50tr;1j~MxEmrosbc7 zLZKl6mN|h{MJP2&GD7(qrYEL<<p5+?B%1IsDHpkX!Ud`Ei~(VcCuCAI84#x;=>7z$ zj8<cWXAGdZL?4Yx;I1Z6jZz(;RLb7fYTY}^6of_NP?n2OOsdcavEscYga@i2oT&P` z!Au5>Fawxe0m2lp5tv~wU@*p@A|)=B#7zq23plJ_L4j^?RD=FcFpdc%O0^sXAxq_G zI6+q_!l{&rL<-!MGNl?g2JBd)EJPxKP(f-?1x5%xoTwmbnpBDlBq+{faWO88GWqr} z8{>&#z66uNT!hPDFeOZcL*Vb$dnmDJeQovY$CQpMF<|3AUd3FT&%;>2keGqOD8gpJ zJPhZ-93GA`xeN({a3<KSRZGEULuKz<)w9BZMFtmVia9(EjH40>%tm=^Sd5}L%oB52 z91e!y1V-rfJSq`^x0D3MiyPk|8;H-xdon3yR22dg)2rQyis{R&k7sOHI!Qi0$n*Gm zZ3F>~{!a17+>lC%MvJNm*Km-#50DrAx9F2-%<q%8M_7CYm*BuCis3L@g7IOL#m8Va z!r+1qit#y|cfn6eo-N=^DEE(%A5RSyfhxiYaDvjQe>|KI)8-h@NyB-6cl|%Zq3h4h zF`d!>TY{r=@13!rn4?4B;MR91UK-$G;<ZC4K&I5-rE+wrL;yiXPC^gYfZ$!VUw0~7 zCFYHv+UymH9YpsoSfzK)-cfB<%y3@nQ+0RoO}5;fqPP*m>a!5t^X;G&??_oDa;x!* zY?-sT+t~J$At!zDU_;rni!qstPwR|C-PJ{ftN35P>>EzF-+!$AsipX}tx4y>&HF=E z*yuuEtNs1d)qP959_RnbXGc17o~}UmOF_3wi;U}<W4%0Ms^@wL>KgoE_>*MHDxPIy zRqNB9BH~(A&vO%|?X~BX&MhkpAN!<ZA<wrT$;`guv+Hc+{FJYtW4%VAmQt@9FOn%L zD9KvLb@Ph2J~+K0cxO?R_kcAe{Y`0g=y8qZ@a0t_5016WjP9t|XBMFCtaW5<vOalk zf!~wWi_R_R&ky=bsErUYcQV_fwzE;$*?44o6j%#WBP%+74$g{GAub)e8^s6nV6*4d z2_fDyichZGRYuLH1=fpBrG$mqT}?CCiod#86v*nWDoY?$TO+V~)%ntZEm_E{e)6Qv z(D%Q5+dl_N<T~DLi+&c#%^fZ+fII_@x4CuY>cYQB;~&`hRZpeKl?7k4^J5S&kx1n) z-ALSyk(qm~<!r3D#d2uB>zu4Ub+Iq~QCsG&Ax4w>yFyk*(U~Wv#%I1GTtkEE6JAAR z%qf3ZQWT~06%?rK(ky27qt0I*S-H{Y-pEI1?X+R`w}YX5>Nn@5c|(5bt<A;7KY4O< zH>H>N9Z$`kJz@$?d9zbL_p+jG59a_;H9RsG)XM7GrKXKR2bf5rxb<vS-{6kHmql+6 z#(idO9%$g%5`MRL>g#Fp286z6*6}-=Lqfbjbt-d_k09TyynXwPtuo9?4#oB6qsoTk z-#abc)-}x)g1Fk(yF)&|U7R=Qu|ByZVEVE~Uuczg*=5<0%%`P?jGUY8H@DXgS-p@) zReV;nEwsJ1*=8s;uzb$bhjCRa%dLQ%S}!%FPzL^f{#mo)OoxIu4^#Dhd*{jaw>zh} zFWUQ~qq)CfQD*iG0JMT4xQ8F)Iwp42zo}C(ZAw=b-y{LHt}E`?;F9ZuCv`HzJC_Y| ztX(%~Y!5fzj|;mHSiV=tZ?-ybmY++obPjCKO7EOR+oY2b?BvwJ6E75X(~^MRmS@~w zk>pl$){<)p0qu3{*IXBC4!-D|Z+O3r%(2$3HL?-r$isB7!6#Ha<*m62+P!aX$6#%x zVKOD@X+lbM;T8wk1<>{<W=~IkP@1IT&1wjqB5JKPoo5QMMO{c1C#B&Bd&5V?SC2Yu zt2@>Gaqxk~w)-s#(v0Y!wY#!cTAfOszFr9cUxK~md*#TQ0c*dvtU){Vlm7B+QN2Gu zX-b^}H8AQI6wMV5Sg)(>Apy<}dQetlZ+))*v3;UrBKuZ6(%-&1&9>MdY?t~V^MyHg zy43BRi~bg53Tkv-(QZ?oX9@l5A4aoH!@7dC<ifez8;(m|qO_a(XI^*j9$4S@>@{BV zNIA0GvZ;<bV7$mTOSeT<uDdtVRF~FW%?;50vy1-C=2w3;S-UKUR0m27D3=BWd$BO- zuvX?H&1gJ#;>S-<taxnmIC57)_y*%|_{O2_Wfnm71`lpMTtkBkGY{3pSAv}sGm{45 z1`qaN?9BWybzN@(Bfj|0Jsr0?59>%Iva+@MTw_^@oxAzc7s5L9?#sg7;Yx@7@@QeL z*ZQ{EmQ8yW70q35z7UWue6^+N?A0|1o<D6Z$h$Gkxw+TnP~KFwyrQS2q#)^N%U1DK v4<N?Vnc3;IwAP@`vM=tas_&XcIgVJAP4U~=|JK4x|7kDu^z$fl+nW4uaj?h8 literal 0 HcmV?d00001 diff --git a/extensions/chrome_ext_v3/background.js b/extensions/chrome_ext_v3/background.js new file mode 100644 index 00000000..0563da23 --- /dev/null +++ b/extensions/chrome_ext_v3/background.js @@ -0,0 +1,35 @@ +/* global chrome */ + +import './deploy/Library/Core/core.js'; +import './deploy/Library/Isolation/vanilla.js'; +//import {Arcs} from './deploy/Library/App/TopLevel/Arcs.js'; +import {App} from './deploy/Library/App/TopLevel/App.js'; + +const paths = { + $library: './deploy/Library' +}; + +(async () => { + await import('./Test.js'); + const app = new App(paths); + //await app.spinup(); + // await Arcs.init({ + // //paths: this.paths, + // //root: this.root || document.body, + // //onservice: this.service.bind(this), + // //injections: {themeRules, ...this.injections} + // }); +})(); + +chrome.action.onClicked.addListener(async () => { + const url = chrome.runtime.getURL("hello.html"); + /*const tab = */await chrome.tabs.create({url}); +}); + +onmessage = event => { + console.log(event); +}; + +chrome.runtime.onMessage.addListener(event => { + console.log(event); +}); \ No newline at end of file diff --git a/extensions/chrome_ext_v3/content-script.js b/extensions/chrome_ext_v3/content-script.js new file mode 100644 index 00000000..e69de29b diff --git a/extensions/chrome_ext_v3/hello.html b/extensions/chrome_ext_v3/hello.html new file mode 100644 index 00000000..d190bf5b --- /dev/null +++ b/extensions/chrome_ext_v3/hello.html @@ -0,0 +1,23 @@ +<!doctype html> + +<link type="text/css" rel="stylesheet" href="./deploy/Library/Dom/Material/material-icon-font/icons.css"> + +<style> + html { + overflow: hidden; + } + body { + margin: 0; + width: 600px; + height: 400px; + overflow: hidden; + display: flex; + flex-direction: column; + align-items: stretch; + } + body > * { + flex: 1; + } +</style> + +<script type="module" src="hello.js"></script> \ No newline at end of file diff --git a/extensions/chrome_ext_v3/hello.js b/extensions/chrome_ext_v3/hello.js new file mode 100644 index 00000000..1825c54c --- /dev/null +++ b/extensions/chrome_ext_v3/hello.js @@ -0,0 +1,13 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import './app/config.js'; +import {quickStart} from './app/arcs.js'; +import {ExtensionApp} from './app/ExtensionApp.js'; + +await quickStart(ExtensionApp, import.meta.url, { + $nodegraph: '$app/deploy/nodegraph/Library' +}); diff --git a/extensions/chrome_ext_v3/jello.html b/extensions/chrome_ext_v3/jello.html new file mode 100644 index 00000000..e8949a8c --- /dev/null +++ b/extensions/chrome_ext_v3/jello.html @@ -0,0 +1,24 @@ +<!doctype html> + +<link type="text/css" rel="stylesheet" href="./deploy/Library/Dom/Material/material-icon-font/icons.css"> + +<style> + html { + overflow: hidden; + } + body { + margin: 0; + width: 600px; + height: 400px; + overflow: hidden; + display: flex; + flex-direction: column; + align-items: stretch; + } + body > * { + flex: 1; + } +</style> + +Jello! +<script type="module" src="jello.js"></script> \ No newline at end of file diff --git a/extensions/chrome_ext_v3/jello.js b/extensions/chrome_ext_v3/jello.js new file mode 100644 index 00000000..1825c54c --- /dev/null +++ b/extensions/chrome_ext_v3/jello.js @@ -0,0 +1,13 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import './app/config.js'; +import {quickStart} from './app/arcs.js'; +import {ExtensionApp} from './app/ExtensionApp.js'; + +await quickStart(ExtensionApp, import.meta.url, { + $nodegraph: '$app/deploy/nodegraph/Library' +}); diff --git a/extensions/chrome_ext_v3/link-deploy.sh b/extensions/chrome_ext_v3/link-deploy.sh new file mode 100755 index 00000000..86b97402 --- /dev/null +++ b/extensions/chrome_ext_v3/link-deploy.sh @@ -0,0 +1,2 @@ +#/bin/sh +ln -s .. deploy \ No newline at end of file diff --git a/extensions/chrome_ext_v3/manifest.json b/extensions/chrome_ext_v3/manifest.json new file mode 100644 index 00000000..8596becd --- /dev/null +++ b/extensions/chrome_ext_v3/manifest.json @@ -0,0 +1,35 @@ +{ + "manifest_version": 3, + "name": "ArcsExtV3", + "version": "0.3.1", + "action": { + "default_icon": { + "32": "assets/logo_32x32.png" + }, + "default_title": "Open Tools", + "default_popup": "jello.html" + }, + "content_scripts": [{ + "matches": ["http://localhost:9888/*"], + "js": ["content-script.js"] + }], + "content_security_policy": { + "sandbox": "sandbox allow-scripts; worker-src blob:; camera 'self';" + }, + "sandbox": { + "pages": [ + "deploy/librarian/index.html", + "viewer.html", + "hello.html" + ] + }, + "background": { + "service_worker": "background.js", + "type": "module" + }, + "permissions": [ + "scripting", + "activeTab", + "storage" + ] +} \ No newline at end of file diff --git a/extensions/video/app/Camera.js b/extensions/video/app/Camera.js new file mode 100644 index 00000000..c0e87c0e --- /dev/null +++ b/extensions/video/app/Camera.js @@ -0,0 +1,34 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +({ +mediaApi(service) { + const media = (msg, data) => service({kind: 'MediaService', msg, data}); + return { + allocateVideo: async () => media('allocateVideo'), + assignStream: async (video, stream) => media('assignStream', {video, stream}), + startFrameCapture: async (video, fps) => media('startFrameCapture', {video, fps}) + } +}, +async initialize(inputs, state, {service}) { + const {allocateVideo} = this.mediaApi(service); + state.video = await allocateVideo('allocateVideo'); +}, +async update({stream, fps}, state, {service, invalidate}) { + const {video, info} = state; + if (info) { + timeout(invalidate, 14); + const frame = {...info.frame, version: Math.random()}; + return {frame}; + } + const {assignStream, startFrameCapture} = this.mediaApi(service); + if (stream) { + await assignStream(video, stream); + state.info = await startFrameCapture(video, fps || 30); + invalidate(); + } +} +}); diff --git a/extensions/video/app/CameraNode.js b/extensions/video/app/CameraNode.js new file mode 100644 index 00000000..2c999baf --- /dev/null +++ b/extensions/video/app/CameraNode.js @@ -0,0 +1,54 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ + import {etc} from './arcs.js'; + + const {DeviceUxRecipe} = etc; + + export const CameraNode = { + $meta: { + id: 'CameraNode', + displayName: "Camera", + category: 'Media' + }, + $stores: { + mediaDevices: DeviceUxRecipe.$stores.mediaDevices, + mediaDeviceState: DeviceUxRecipe.$stores.mediaDeviceState, + stream: { + $type: 'Stream', + $value: 'default' + }, + fps: { + $type: 'Number', + $value: 30 + }, + frame: { + $type: 'Image', + noinspect: true, + nomonitor: true + } + }, + camera: { + $kind: '$app/Camera', + $staticInputs: { + stream: 'default' + }, + $outputs: ['stream', 'frame'], + $slots: { + device: { + deviceUx: DeviceUxRecipe.deviceUx, + defaultStream: DeviceUxRecipe.defaultStream + }, + capture: { + imageCapture: { + $kind: '$library/NewMedia/ImageCapture', + $inputs: ['stream', 'fps'], + $outputs: ['frame'] + } + } + } + } + }; \ No newline at end of file diff --git a/extensions/video/app/ExtApp.js b/extensions/video/app/ExtApp.js new file mode 100644 index 00000000..4dd071fc --- /dev/null +++ b/extensions/video/app/ExtApp.js @@ -0,0 +1,25 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import { + Paths, App, LocalStoragePersistor, + HistoryService, + ThreejsService, + ShaderService, + TensorFlowService, + SelfieSegmentationService, + MediaService +} from './arcs.js'; +import {ExtRecipe} from './ExtRecipe.js'; + +export const ExtApp = class extends App { + constructor() { + super(Paths.map); + this.services = {HistoryService, ThreejsService, ShaderService, TensorFlowService, SelfieSegmentationService, MediaService}; + this.userAssembly = [ExtRecipe]; + this.persistor = new LocalStoragePersistor('user'); + } +}; diff --git a/extensions/video/app/ExtRecipe.js b/extensions/video/app/ExtRecipe.js new file mode 100644 index 00000000..d3ee2c3c --- /dev/null +++ b/extensions/video/app/ExtRecipe.js @@ -0,0 +1,116 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +// import {etc} from "./arcs.js"; +import {giantInLake} from "../assets/giantInLake.js"; + +export const ExtRecipe = { + $stores: { + "camera1:input": { + $type: 'Image' + }, + "deviceimage1:output": { + $type: 'Image' + }, + "shaderFrame": { + $type: 'Image' + }, + "selfieFrame": { + $type: 'Image' + }, + "composedFrame1": { + $type: 'Image' + }, + "composedFrame2": { + $type: 'Image' + }, + "frame": { + $type: 'Image' + }, + "pixiFrame": { + $type: 'Image' + } + }, + deviceimage1: { + $kind: '$library/NewMedia/DeviceImage.js', + $inputs: [{image: 'frame'}], + $outputs: [{output: 'deviceimage1:output'}] + }, + // + // noop: { + // $kind: '$library/Noop', + // // $staticInputs: { + // // html: `<div frame="device" style="display: none;"></div>` + // // }, + // $slots: { + // device: etc.DeviceUxRecipe + // } + // }, + // camera: { + // $kind: '$app/Camera', + // $inputs: ['stream', 'fps'], + // // $outputs: ['frame'] + // $outputs: ['deviceimage1:output'] + // }, + // + // + pixi: { + $kind: '$library/PixiJs/PixiJs', + $staticInputs: { + demo: 'Spiral' + }, + $outputs: [{ + image: 'pixiFrame' + }] + }, + SelfieSegmentation: { + $kind: 'Mediapipe/SelfieSegmentation', + $inputs: [ + {'image': 'camera1:input'} + ], + $outputs: [ + {'mask': 'selfieFrame'} + ] + }, + compose1: { + $kind: '$library/NewMedia/ImageComposite', + $staticInputs: { + operation: 'overlay' + }, + $inputs: [ + {imageA: 'camera1:input'}, + {imageB: 'pixiFrame'} + ], + $outputs: [ + {output: 'composedFrame1'} + ] + }, + compose2: { + $kind: '$library/NewMedia/ImageComposite', + $staticInputs: { + operation: 'source-over' + }, + $inputs: [ + {imageA: 'composedFrame1'}, + {imageB: 'selfieFrame'} + ], + $outputs: [ + {output: 'composedFrame2'} + ] + }, + shader: { + $kind: '$library/Shader/FragmentShader', + $staticInputs: { + shader: giantInLake + }, + $inputs: [ + {'image': 'composedFrame2'} + ], + $outputs: [ + {'outputImage': 'frame'} + ] + } +}; diff --git a/extensions/video/app/Resources.js b/extensions/video/app/Resources.js new file mode 100644 index 00000000..09cbb05b --- /dev/null +++ b/extensions/video/app/Resources.js @@ -0,0 +1,29 @@ +/** + * @license + * Copyright 2022 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ +import {Resources} from './arcs.js'; + +let resources = {}; + +Object.assign(Resources, { + use(_resources) { + resources = _resources; + }, + get(id) { + return resources[id]; + }, + set(id, resource) { + resources[id] = resource; + return id; + }, + all() { + return resources; + } +}); + +export {Resources}; \ No newline at end of file diff --git a/extensions/video/app/arcs.js b/extensions/video/app/arcs.js new file mode 100644 index 00000000..2b2ebb6e --- /dev/null +++ b/extensions/video/app/arcs.js @@ -0,0 +1,64 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ + +// use Library path from configuration + +const Library = `${globalThis.config.arcsPath}/Library`; + +// import modules from the ArcsJs Library +// the 'load' function imports modules in parallel + +const load = async paths => (await Promise.all(paths.map(p => import(`${Library}/${p}`)))).reduce((e, m) =>({...e, ...m}),{}); + +export const { + Paths, logFactory, + App, Resources, Params, + Xen, deepQuerySelector, + quickStart, + LocalStoragePersistor, + HistoryService, MediaService, + TensorFlowService, + FaceMeshService, SelfieSegmentationService, + MediapipeNodes, + ThreejsService, ShaderService, + CocoSsdService, + LobbyService, + // must be last + ...etc +} = await load([ + 'Core/utils.js', + // + 'App/Worker/App.js', + 'App/Resources.js', + 'App/Params.js', + 'App/HistoryService.js', + 'App/boot.js', + // + 'App/common-dom.js', + 'Designer/designer-layout.js', + // + 'PixiJs/pixi-view.js', + // + 'LocalStorage/LocalStoragePersistor.js', + 'Media/DeviceUxRecipe.js', + // + 'TensorFlow/TensorFlow.js', + 'TensorFlow/TensorFlowService.js', + 'Rtc/LobbyService.js', + 'NewMedia/MediaService.js', + 'Mediapipe/FaceMeshService.js', + 'Mediapipe/SelfieSegmentationService.js', + 'Threejs/ThreejsService.js', + 'Shader/ShaderService.js', + 'TensorFlow/CocoSsdService.js', + // + 'GraphsNodes/CameraNode.js', + 'Shader/ShaderNodes.js', + 'Mediapipe/MediapipeNodes.js', + 'TensorFlow/CocoSsdNode.js', + 'Display/DisplayNodes.js' +]); diff --git a/extensions/video/app/boot.js b/extensions/video/app/boot.js new file mode 100644 index 00000000..64fd65e3 --- /dev/null +++ b/extensions/video/app/boot.js @@ -0,0 +1,9 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import "./index.js"; + +globalThis.app.arcs.setComposerRoot(document.body); diff --git a/extensions/video/app/config.js b/extensions/video/app/config.js new file mode 100644 index 00000000..82056b5b --- /dev/null +++ b/extensions/video/app/config.js @@ -0,0 +1,29 @@ +globalThis.config = { + aeon: 'ExtVideo/0.4.4', + // + // use for normal loading + //arcsPath: 'https://arcsjs.web.app/0.4.4', + // use for arcsjs:serve debugging + arcsPath: 'http://localhost:9888/0.4.4', + // use for symlink ArcsJs debugging + //arcsPath: '/env/arcsjs', + // other options... + //arcsPath: '../../../arcsjs-core/pkg', + // + logFlags: { + app: true, + //arc: true, + //bus: true, + //composer: true, + //isolation: true, + //media: true, + particles: true, + //recipe: true, + //services: true, + //storage: true, + //worker: true, + //ShaderService: true + //Studio: true + //TfjsService: true + } +}; diff --git a/extensions/video/app/index.html b/extensions/video/app/index.html new file mode 100644 index 00000000..fc766fb9 --- /dev/null +++ b/extensions/video/app/index.html @@ -0,0 +1,11 @@ +<!doctype html> +<!-- /** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ --> +<script src="https://arcsjs.web.app/lib/corsFix.js"></script> +<script src="../../../third_party/pixijs/pixi.6.5.7.min.js"></script> +<script src="../../../third_party/pixijs/pixi-plugins/pixi-spine.js"></script> +<script type="module" src="boot.js"></script> diff --git a/extensions/video/app/index.js b/extensions/video/app/index.js new file mode 100644 index 00000000..f3aa2988 --- /dev/null +++ b/extensions/video/app/index.js @@ -0,0 +1,11 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import './config.js'; +import {ExtApp} from './ExtApp.js'; +import {quickStart} from './arcs.js'; + +await quickStart(ExtApp, import.meta.url); diff --git a/extensions/video/assets/giantInLake.js b/extensions/video/assets/giantInLake.js new file mode 100644 index 00000000..61fa2da1 --- /dev/null +++ b/extensions/video/assets/giantInLake.js @@ -0,0 +1,34 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +export const giantInLake = ` +/* +* Webcam 'Giant in a lake' effect by Ben Wheatley - 2018 +* License MIT License +* Contact: github.com/BenWheatley +*/ +void mainImage( out vec4 fragColor, in vec2 fragCoord ) +{ + float time = iTime; + vec2 uv = fragCoord.xy / iResolution.xy; + vec2 pixelSize = vec2(1,1) / iResolution.xy; + vec3 col = texture(iChannel0, uv).rgb; + float mirrorPos = 0.3; + if (uv.y < mirrorPos) { + float distanceFromMirror = mirrorPos - uv.y; + float sine = sin((log(distanceFromMirror)*20.0) + (iTime*2.0)); + float dy = 30.0*sine; + float dx = 0.0; + dy *= distanceFromMirror; + vec2 pixelOff = pixelSize * vec2(dx, dy); + vec2 tex_uv = uv + pixelOff; + tex_uv.y = (0.6 /* magic number! */) - tex_uv.y; + col = texture(iChannel0, tex_uv).rgb; + float shine = (sine + dx*0.05) * 0.05; + col += vec3(shine, shine, shine); + } + fragColor = vec4(col,1.); +}`; \ No newline at end of file diff --git a/extensions/video/assets/logo_32x32.png b/extensions/video/assets/logo_32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..13a239202934879d8b432d4c639cdced3544ddb2 GIT binary patch literal 3127 zcmb_e3se(V8XmCtKtV;0XvH=}6cr|q$%7=k<S9Xdihxi>oXkuhkXJK-@X{h8$V0JE zt@uPi++_iEIfy`k-J(#`stAff5o>)c%PLZ+h*a4dK-TV|T~E*EWbWL#_q+f9{r~^n zJCnZ7Z?%c>$Hou@nFzf-{K1`~zot(GEg2s_0B$o>-XSCe&77yd44^|t7C_L{d}%<i zCRpSvz?5<tiYvtgEmp1qXb5t2iB+Lk6rrJriEydHiP~FJPo+q4C+bFyh#^wB6A@DH zcr~#h-Y)=)kHYvk)y0|O7%Kn<<b(#L#L8s~QV{Dz9knX}ef=<<N*RS{qMWF%dWV!? z(K?E|QcY0oX$Xul7z~O%pT-n(IjB8f%(kN-3<RMwxO659Mi2pm3I3Gv9~C%M;}U_t zhu63}@Z>~|&}dWwI$f*P(zGm^QXNib^7(u^1EC`b3=l9Gr_i9Wu!3Cj&VmO)Vrr>M zBULIWdW)!78Le@m0<vRK$W;?&736rPK+fo~sEW>{G4xT50tr;1j~MxEmrosbc7 zLZKl6mN|h{MJP2&GD7(qrYEL<<p5+?B%1IsDHpkX!Ud`Ei~(VcCuCAI84#x;=>7z$ zj8<cWXAGdZL?4Yx;I1Z6jZz(;RLb7fYTY}^6of_NP?n2OOsdcavEscYga@i2oT&P` z!Au5>Fawxe0m2lp5tv~wU@*p@A|)=B#7zq23plJ_L4j^?RD=FcFpdc%O0^sXAxq_G zI6+q_!l{&rL<-!MGNl?g2JBd)EJPxKP(f-?1x5%xoTwmbnpBDlBq+{faWO88GWqr} z8{>&#z66uNT!hPDFeOZcL*Vb$dnmDJeQovY$CQpMF<|3AUd3FT&%;>2keGqOD8gpJ zJPhZ-93GA`xeN({a3<KSRZGEULuKz<)w9BZMFtmVia9(EjH40>%tm=^Sd5}L%oB52 z91e!y1V-rfJSq`^x0D3MiyPk|8;H-xdon3yR22dg)2rQyis{R&k7sOHI!Qi0$n*Gm zZ3F>~{!a17+>lC%MvJNm*Km-#50DrAx9F2-%<q%8M_7CYm*BuCis3L@g7IOL#m8Va z!r+1qit#y|cfn6eo-N=^DEE(%A5RSyfhxiYaDvjQe>|KI)8-h@NyB-6cl|%Zq3h4h zF`d!>TY{r=@13!rn4?4B;MR91UK-$G;<ZC4K&I5-rE+wrL;yiXPC^gYfZ$!VUw0~7 zCFYHv+UymH9YpsoSfzK)-cfB<%y3@nQ+0RoO}5;fqPP*m>a!5t^X;G&??_oDa;x!* zY?-sT+t~J$At!zDU_;rni!qstPwR|C-PJ{ftN35P>>EzF-+!$AsipX}tx4y>&HF=E z*yuuEtNs1d)qP959_RnbXGc17o~}UmOF_3wi;U}<W4%0Ms^@wL>KgoE_>*MHDxPIy zRqNB9BH~(A&vO%|?X~BX&MhkpAN!<ZA<wrT$;`guv+Hc+{FJYtW4%VAmQt@9FOn%L zD9KvLb@Ph2J~+K0cxO?R_kcAe{Y`0g=y8qZ@a0t_5016WjP9t|XBMFCtaW5<vOalk zf!~wWi_R_R&ky=bsErUYcQV_fwzE;$*?44o6j%#WBP%+74$g{GAub)e8^s6nV6*4d z2_fDyichZGRYuLH1=fpBrG$mqT}?CCiod#86v*nWDoY?$TO+V~)%ntZEm_E{e)6Qv z(D%Q5+dl_N<T~DLi+&c#%^fZ+fII_@x4CuY>cYQB;~&`hRZpeKl?7k4^J5S&kx1n) z-ALSyk(qm~<!r3D#d2uB>zu4Ub+Iq~QCsG&Ax4w>yFyk*(U~Wv#%I1GTtkEE6JAAR z%qf3ZQWT~06%?rK(ky27qt0I*S-H{Y-pEI1?X+R`w}YX5>Nn@5c|(5bt<A;7KY4O< zH>H>N9Z$`kJz@$?d9zbL_p+jG59a_;H9RsG)XM7GrKXKR2bf5rxb<vS-{6kHmql+6 z#(idO9%$g%5`MRL>g#Fp286z6*6}-=Lqfbjbt-d_k09TyynXwPtuo9?4#oB6qsoTk z-#abc)-}x)g1Fk(yF)&|U7R=Qu|ByZVEVE~Uuczg*=5<0%%`P?jGUY8H@DXgS-p@) zReV;nEwsJ1*=8s;uzb$bhjCRa%dLQ%S}!%FPzL^f{#mo)OoxIu4^#Dhd*{jaw>zh} zFWUQ~qq)CfQD*iG0JMT4xQ8F)Iwp42zo}C(ZAw=b-y{LHt}E`?;F9ZuCv`HzJC_Y| ztX(%~Y!5fzj|;mHSiV=tZ?-ybmY++obPjCKO7EOR+oY2b?BvwJ6E75X(~^MRmS@~w zk>pl$){<)p0qu3{*IXBC4!-D|Z+O3r%(2$3HL?-r$isB7!6#Ha<*m62+P!aX$6#%x zVKOD@X+lbM;T8wk1<>{<W=~IkP@1IT&1wjqB5JKPoo5QMMO{c1C#B&Bd&5V?SC2Yu zt2@>Gaqxk~w)-s#(v0Y!wY#!cTAfOszFr9cUxK~md*#TQ0c*dvtU){Vlm7B+QN2Gu zX-b^}H8AQI6wMV5Sg)(>Apy<}dQetlZ+))*v3;UrBKuZ6(%-&1&9>MdY?t~V^MyHg zy43BRi~bg53Tkv-(QZ?oX9@l5A4aoH!@7dC<ifez8;(m|qO_a(XI^*j9$4S@>@{BV zNIA0GvZ;<bV7$mTOSeT<uDdtVRF~FW%?;50vy1-C=2w3;S-UKUR0m27D3=BWd$BO- zuvX?H&1gJ#;>S-<taxnmIC57)_y*%|_{O2_Wfnm71`lpMTtkBkGY{3pSAv}sGm{45 z1`qaN?9BWybzN@(Bfj|0Jsr0?59>%Iva+@MTw_^@oxAzc7s5L9?#sg7;Yx@7@@QeL z*ZQ{EmQ8yW70q35z7UWue6^+N?A0|1o<D6Z$h$Gkxw+TnP~KFwyrQS2q#)^N%U1DK v4<N?Vnc3;IwAP@`vM=tas_&XcIgVJAP4U~=|JK4x|7kDu^z$fl+nW4uaj?h8 literal 0 HcmV?d00001 diff --git a/extensions/video/background.html b/extensions/video/background.html new file mode 100644 index 00000000..3e546b55 --- /dev/null +++ b/extensions/video/background.html @@ -0,0 +1,10 @@ +<!doctype html> +<!-- + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. +--> +<meta charset="utf-8"> + +<script type="module" src="./plugin/background.js"></script> \ No newline at end of file diff --git a/extensions/video/manifest.json b/extensions/video/manifest.json new file mode 100644 index 00000000..5347dbef --- /dev/null +++ b/extensions/video/manifest.json @@ -0,0 +1,40 @@ +{ + "manifest_version": 2, + "name": "ArcsVideov2_1", + "description": "Arcs Video Extension v2.1", + "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs8wiwzvpbAdK5eMfIofO0aKo9lOZ0XUOdzOGJzgvVs0EWavUvzhu5BQwT4yr4+RmjSog2GVMnVnYA0pz2V5TMuZzxt9VD9FLRKL/IBM2GImnmKmBgee0jRVmJkGXvxyy9g9ruj/0TfE4S4GzEwD6ckT85mWEmY1eUuH2Z+LNScBvtj7ckpSeA98sc7MhRBBFJdcrlG+AAvf4MyYzJyswoUt6MSyJybu3nyNQLeNvPbntWxwOwokfxZ8YHZQUzdMfm7VDAV8LVGbmwYFTWnSXxkj4wkmRn2uIqzDr9Dnb7NxySFLFiZGx+mf9vrXtVx6vAs+7AvocK8r+lAUghGLcfQIDAQAB", + "version": "2.1.0", + "permissions": [ + "activeTab", + "storage" + ], + "background": { + "page": "background.html", + "persistent": true + }, + "browser_action": { + "default_icon": "assets/logo_32x32.png", + "default_title": "Arcs Video" + }, + "options_ui": { + "page": "options.html", + "open_in_tab": true + }, + "icons": { + "32": "assets/logo_32x32.png" + }, + "content_scripts": [{ + "matches": [ + "http://localhost:9888/*", + "https://arcsjs.web.app/*", + "https://meet.google.com/*" + ], + "js": ["plugin/content.js"], + "run_at": "document_start" + }], + "content_security_policy": "script-src 'self' 'unsafe-eval' http://localhost:9888/; object-src 'self'", + "web_accessible_resources": [ + "plugin/virtual-camera.js", + "plugin/virtual-stream.js" + ] +} diff --git a/extensions/video/options.html b/extensions/video/options.html new file mode 100644 index 00000000..934abfe4 --- /dev/null +++ b/extensions/video/options.html @@ -0,0 +1,15 @@ +<!-- + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. +--> +<!doctype html> +<body> + +<h2>I'm here for the DevTools.</h2> +<resource-view></resource-view> + +<script type="module" src="./plugin/options.js"></script> + +</body> \ No newline at end of file diff --git a/extensions/video/plugin/background.js b/extensions/video/plugin/background.js new file mode 100644 index 00000000..c281abef --- /dev/null +++ b/extensions/video/plugin/background.js @@ -0,0 +1,11 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +/* global chrome */ +import {initRtc} from './web-rtc.js'; +import {getArcsCameraStream} from './cameras/arcs-camera.js'; + +initRtc(getArcsCameraStream); diff --git a/extensions/video/plugin/cameras/arcs-camera.js b/extensions/video/plugin/cameras/arcs-camera.js new file mode 100644 index 00000000..96c41415 --- /dev/null +++ b/extensions/video/plugin/cameras/arcs-camera.js @@ -0,0 +1,91 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import '../../app/config.js'; +import {Resources} from '../../app/arcs.js'; +// TODO(sjmiles): weird place to boot the app +import '../../app/index.js'; + +const dom = (tag, props) => document.body.appendChild(Object.assign(document.createElement(tag), props)); + +export const animateCanvas = async (fps, update) => { + let frame = 0; + const interval = Math.max(0, Math.round(1000 / fps)); + setInterval(() => { + frame = (frame + 1) % 3; + update(frame); + }, interval); +} + +const flashGray = (ctx, frame) => { + const {width, height} = ctx.canvas; + ctx.fillStyle = ['#88888820', '#88888840', '#88888830'][frame]; + ctx.fillRect(20, 20, width - 20, height - 20); +}; + +const snapVideo = (video, ctx, frame) => { + const {width, height} = ctx.canvas; + ctx.drawImage(video, 0, 0, width, height); + // TODO(sjmiles): this works, but it's REALLY SLOW (!?) + //ctx.scale(1, -1); + //ctx.drawImage(video, 0, -height); + //ctx.drawImage(video, 0, 0); +}; + +// for debugging +globalThis.Resources = Resources; + +const imageInputStore = `camera1:input`; +const canvasOutputStore = `deviceimage1:output`; + +// export const arcsProcessImage = async (inputCanvasId, outputCtx, frame) => { +// //console.log('get device output canvas'); +// app.arcs.set('user', imageInputStore, {canvas: inputCanvasId, version: Math.random()}); +// // // +// // const imageRef = await app.arcs.get('user', canvasOutputStore); +// // const arcsCanvas = Resources.get(imageRef?.canvas); +// // // +// // if (arcsCanvas && outputCtx) { +// // outputCtx.drawImage(arcsCanvas, 0, 0); +// // } +// }; + +export const getArcsCameraStream = async (stream) => { + const video = dom('video'); + video.srcObject = stream ?? await navigator.mediaDevices.getUserMedia({video: {}}); + await video.play(); + // + const inCanvas = dom('canvas', {width: 1280, height: 720, style: 'position: absolute;'}); + const inCtx = inCanvas.getContext('2d'); + const inputCanvasId = globalThis.Resources.allocate(inCanvas); + // + const outCanvas = dom('canvas', {width: 1280, height: 720, style: 'position: absolute;'}); + const outCtx = outCanvas.getContext('2d'); + // + const fps = 30; + // + let arcsResultCanvas; + // + const update = (frame) => { + snapVideo(video, inCtx); + app.arcs.set('user', imageInputStore, {canvas: inputCanvasId, version: Math.random()}); + if (inCanvas) { + outCtx.drawImage(inCanvas, 0, 0); + } + if (arcsResultCanvas) { + outCtx.drawImage(arcsResultCanvas, 0, 0); + } else { + (async () => { + const imageRef = await app.arcs.get('user', canvasOutputStore); + arcsResultCanvas = Resources.get(imageRef?.canvas); + })(); + } + //flashGray(outCtx, frame); + }; + animateCanvas(fps, update); + // + return outCanvas.captureStream(fps); +}; diff --git a/extensions/video/plugin/cameras/basic-camera.js b/extensions/video/plugin/cameras/basic-camera.js new file mode 100644 index 00000000..9cd0c460 --- /dev/null +++ b/extensions/video/plugin/cameras/basic-camera.js @@ -0,0 +1,63 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +const FRAME_SIZE = {width: 1280, height: 720}; + +const VIDEO = document.createElement('video'); +VIDEO.id = 'basic-camera-video'; + +// Input canvas +//const INPUT_CAMERA_CANVAS = new OffscreenCanvas(FRAME_SIZE.width, FRAME_SIZE.height); +const INPUT_CAMERA_CANVAS = document.createElement('canvas'); +Object.assign(INPUT_CAMERA_CANVAS, {...FRAME_SIZE, id: 'basic-camera-input-canvas'}); +const INPUT_CAMERA_CTX = INPUT_CAMERA_CANVAS.getContext('2d'); + +// Output canvas +const OUTPUT_CANVAS = document.createElement('canvas'); +Object.assign(OUTPUT_CANVAS, {...FRAME_SIZE, id: 'basic-camera-output-canvas'}); + +/** + * Copies incoming image from real camera to the input camera canvas + */ +let capturing; +async function captureLoop() { + if (!capturing) { + capturing = true; + await capture(); + capturing = false; + } +} + +/** + * Enumerates through required models and run predictions + */ +async function capture() { + INPUT_CAMERA_CTX.drawImage(VIDEO, 0, 0); +} + +let loopInterval; +async function start() { + const cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { + ...FRAME_SIZE + } + }); + VIDEO.srcObject = cameraStream; + await VIDEO.play(); + // + window.clearInterval(loopInterval); + loopInterval = window.setInterval(captureLoop, 16); + // + return INPUT_CAMERA_CANVAS; +} + +export const getBasicCameraStream = async () => { + const canvas = await start(); + console.log(canvas); + const stream = canvas.captureStream(30); + console.log(stream); + return stream; +}; \ No newline at end of file diff --git a/extensions/video/plugin/cameras/x-arcs-camera.js b/extensions/video/plugin/cameras/x-arcs-camera.js new file mode 100644 index 00000000..8f0593f6 --- /dev/null +++ b/extensions/video/plugin/cameras/x-arcs-camera.js @@ -0,0 +1,47 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +import {Resources} from '../arcs/index.js'; + +// for debugging +globalThis.Resources = Resources; + +const imageInputStore = `camera1:input`; +const canvasOutputStore = `deviceimage1:output`; + +export const arcsProcessImage = async (inputCanvasId, outputCtx, frame) => { + //console.log('get device output canvas'); + app.arcs.set('user', imageInputStore, {canvas: inputCanvasId, version: Math.random()}); + // + const imageRef = await app.arcs.get('user', canvasOutputStore); + const arcsCanvas = Resources.get(imageRef?.canvas); + // + if (arcsCanvas && outputCtx) { + outputCtx.drawImage(arcsCanvas, 0, 0); + } +}; + +// export const getArcsCameraStream = async () => new Promise(resolve => { +// console.log('wait for arc spin up'); +// setTimeout(() => { +// console.log('setting mediaDeviceState'); +// app.arcs.set('user', 'mediaDeviceState', {isCameraEnabled: true, isMicEnabled: false, isAudioEnabled: false}); +// console.log('wait for media spin up'); +// setTimeout(async () => { +// console.log('get device output canvas'); +// const imageRef = await app.arcs.get('user', canvasOutputStore); +// const canvas = Resources.get(imageRef.canvas); +// const stream = canvas.captureStream(30); +// console.log(imageRef, stream); +// // console.log('get stream data'); +// // const streamId = await app.arcs.get('user', 'stream'); +// // const stream = Resources.get(streamId); +// // console.log(streamId, stream); +// // resolve with stream +// resolve(stream); +// }, 5000) +// }, 2000); +// }); \ No newline at end of file diff --git a/extensions/video/plugin/content.js b/extensions/video/plugin/content.js new file mode 100644 index 00000000..a6e72075 --- /dev/null +++ b/extensions/video/plugin/content.js @@ -0,0 +1,31 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ + +// load code into the client's context (try to keep it neat!) +const loadInPage = src => document.firstElementChild.appendChild(Object.assign(document.createElement('script'), {src})); +loadInPage(chrome.runtime.getURL('plugin/virtual-camera.js')); +loadInPage(chrome.runtime.getURL('plugin/virtual-stream.js')); + +// connect to the extension +const port = chrome.runtime.connect(); +//const port = chrome.runtime.connect({name: 'client'}); + +// forward messages from the extension to the client window +const forwardToClient = msg => { + console.log("(forwarding to client)", msg.type); + window.postMessage({...msg, incoming: true}, '*'); +}; +port.onMessage.addListener(forwardToClient); + +// forward messages from the client window to the extension +const forwardToExtension = ({data: msg}) => { + if (msg.outgoing) { + console.log("(forwarding to extension)", msg.type); + port.postMessage(msg); + } +}; +window.addEventListener('message', forwardToExtension); \ No newline at end of file diff --git a/extensions/video/plugin/options.js b/extensions/video/plugin/options.js new file mode 100644 index 00000000..99393536 --- /dev/null +++ b/extensions/video/plugin/options.js @@ -0,0 +1,18 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +/* global chrome */ +import '../app/config.js'; +import {Resources} from '../app/arcs.js'; + +// get extreme powers +chrome.runtime.getBackgroundPage(bgPage => { + const {app, Resources: bgResources} = bgPage; + // share resources with bgPage + Resources.use(bgResources.all()); + // render here + app.arcs.setComposerRoot(document.body); +}); diff --git a/extensions/video/plugin/virtual-camera.js b/extensions/video/plugin/virtual-camera.js new file mode 100644 index 00000000..57be8609 --- /dev/null +++ b/extensions/video/plugin/virtual-camera.js @@ -0,0 +1,68 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +const builtin = navigator.mediaDevices; +const fallback = MediaDevices.prototype; + +let asyncStreamGetter; + +globalThis.createVirtualCamera = _asyncStreamGetter => { + asyncStreamGetter = _asyncStreamGetter; + builtin.getUserMedia = getUserMedia; + builtin.enumerateDevices = enumerateDevices; + builtin.dispatchEvent( + new CustomEvent('devicechange') + ); +}; + +const enumerateDevices = async function () { + const devices = await fallback.enumerateDevices.call(this); + const virtualCamera = { + deviceId: 'virtual', + groupID: 'ArcsJs', + kind: 'videoinput', + label: 'ArcsJs Virtual Camera' + }; + return [...devices, virtualCamera]; +}; + +const getUserMedia = async function (constraints) { + if (constraints) { + const videoDeviceId = getDeviceId(constraints.video); + if (videoDeviceId === 'virtual') { + return marshalVirtualStream(constraints); + } else { + return fallback.getUserMedia.call(this, constraints); + } + } +}; + +const marshalVirtualStream = async ({audio, video}) => { + const stream = await asyncStreamGetter(video); + if (audio) { + const audioStream = await fallback.getUserMedia.call(this, {audio, video: false}); + for (const track of audioStream.getAudioTracks()) { + stream.addTrack(track); + } + } + return stream; +}; + +const getDeviceId = videoConstraints => { + if (typeof videoConstraints === 'boolean') { + return null; + } + if (!videoConstraints?.deviceId) { + return null; + } + if (typeof videoConstraints.deviceId === 'string') { + return videoConstraints.deviceId; + } + if (videoConstraints.deviceId instanceof Array) { + return videoConstraints.deviceId[0]; + } + return videoConstraints.deviceId.exact ?? null; +}; diff --git a/extensions/video/plugin/virtual-stream.js b/extensions/video/plugin/virtual-stream.js new file mode 100644 index 00000000..54d56dcb --- /dev/null +++ b/extensions/video/plugin/virtual-stream.js @@ -0,0 +1,97 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +let virtualTrack; +let signature; +let connection; + +const send = (type, props) => { + console.log('virtual-stream: sending', type, props); + window.postMessage({type, ...props, outgoing: true, signature}, '*'); +}; + +const getVirtualMediaStream = async (constraints) => { + if (!virtualTrack) { + virtualTrack = await createVirtualTrack(constraints); + } + if (virtualTrack) { + const stream = new MediaStream(); + stream.addTrack(virtualTrack); + return stream; + } +}; + +const createVirtualTrack = async (constraints) => { + return await new Promise(resolve => { + // make a signature out of some entropy + signature = Math.trunc(performance.now()); + // create an actual rtc thing + connection = new RTCPeerConnection(); + // rtc listeners + connection.addEventListener('icecandidate', onIceCandidate); + connection.addEventListener('track', e => onTrack(resolve, e)); + // command listener + window.addEventListener('message', messageListener); + // send request to background + console.log('sending GET_STREAM'); + send('GET_STREAM', {constraints}); + }); +}; + +const messageListener = async ({data: msg}) => { + if (msg.incoming && msg.signature === signature) { + console.log('virtual-stream received:', msg.type); + switch (msg.type) { + case 'RTC_OFFER': + return acceptOffer(msg); + case 'RTC_ICE': + return acceptIce(msg); + } + } +}; + +const acceptOffer = async ({description}) => { + await connection.setRemoteDescription(description); + const answer = await connection.createAnswer(); + await connection.setLocalDescription(answer); + send('RTC_ANSWER', {description: new RTCSessionDescription(answer).toJSON()}); +}; + +const acceptIce = async ({candidate}) => { + await connection.addIceCandidate(candidate); +}; + +const onIceCandidate = (event) => { + if (event.candidate) { + const candidate = event.candidate.toJSON(); + send('RTC_ICE', {candidate}); + } +}; + +const onTrack = (resolve, event) => { + console.log('onTrack', event); + // why here?? + //window.removeEventListener('message', windowMessageListener); + // get a new track + const track = event.track; + // mess with the API (adding or modifying?) + track.applyConstraints = async () => {}; + track.clone = () => track; + track.stop = () => { + // I guess it's not a real boy? + MediaStreamTrack.prototype.stop.call(track); + virtualTrack = null; + // stop track === close connection + connection.close(); + // notify upstream + send('CLOSE_STREAM'); + } + console.log('onTrack resolving with:', track); + // finally, here's the track! + resolve(track); +}; + +globalThis.createVirtualCamera(getVirtualMediaStream); diff --git a/extensions/video/plugin/web-rtc.js b/extensions/video/plugin/web-rtc.js new file mode 100644 index 00000000..e8eabd47 --- /dev/null +++ b/extensions/video/plugin/web-rtc.js @@ -0,0 +1,95 @@ +/** + * @license + * Copyright (c) 2022 Google LLC All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ + +let connections = {}; + +export const initRtc = (requestMediaStream) => { + chrome.runtime.onConnect.addListener(port => { + console.log('web-rtc: port connected'); + // Most likely client tab was closed. + port.onDisconnect.addListener(() => { + console.log('web-rtc: port disconnect'); + //globalClientCounter -= rtcConnections.size; + connections = {}; + //maybeStopEngine(); + }); + port.onMessage.addListener(async message => { + //console.log('web-rtc: received', message); + // Client can send an optional signature. This way we can distinguish + // between different camera requests. If unset, it defaults to 0. + const signature = message.signature ?? 0; + const connection = connections[signature]; + switch (message.type) { + // stream is requested + case 'GET_STREAM': + getStream(port, requestMediaStream, signature); + break; + // received an answer + case 'RTC_ANSWER': + console.log('web-rtc: received', message); + await connection?.setRemoteDescription(message.description); + break; + // received ICE candidate info + case 'RTC_ICE': + console.log('web-rtc: received', message); + await connection?.addIceCandidate(message.candidate); + break; + // cient closed the stream + case 'CLOSE_STREAM': + if (connection) { + connection.close(); + connections[signature] = null; + } + break; + } + }); + }); +}; + +const offerOptions = { + offerToReceiveVideo: true, +}; + +const getStream = async (port, requestMediaStream, signature) => { + //console.log('web-rtc: creating stream connection'); + const stream = await requestMediaStream(); + // create connection + const connection = new RTCPeerConnection(); + connections[signature] = connection; + //console.log(signature, connection); + // + // add output stream tracks + for (const track of stream.getTracks()) { + //console.log('web-rtc: added a track', track); + connection.addTrack(track); + } + // + // create offer + const offer = await connection.createOffer(offerOptions); + await connection.setLocalDescription(offer); + //console.log('web-rtc: created an offer'); + // + // forward ICE candidate info when it becomes available + connection.addEventListener('icecandidate', (event) => { + //console.log('web-rtc: received icecandidate', event.candidate); + if (event.candidate) { + rtcIce(port, event.candidate, signature); + } + }); + //console.log('web-rtc: sent an offer', offer); + // send offer + rtcOffer(port, offer, signature); +}; + +const msg = (port, type, fields) => { + const msg = {type, ...fields}; + //console.log('web-rtc: sending to client:', msg); + port.postMessage(msg); +}; + +const rtcIce = (port, candidate, signature) => msg(port, 'RTC_ICE', {candidate, signature}); +const rtcOffer = (port, description, signature) => msg(port, 'RTC_OFFER', {description, signature});