From 5a92717b811e0679369004ae6d554225178be373 Mon Sep 17 00:00:00 2001 From: Latif Date: Thu, 15 Feb 2024 08:51:45 -0700 Subject: [PATCH 01/13] support for config --- LocalFeeder/server.py | 41 +++++++++++++++++++++--------- broker/server.py | 49 ++++++++++++++++++++++++------------ measuring_federate/server.py | 31 +++++++++++++++++------ recorder/server.py | 34 ++++++++++++++++++++----- wls_federate/server.py | 26 ++++++++++++++++--- 5 files changed, 135 insertions(+), 46 deletions(-) diff --git a/LocalFeeder/server.py b/LocalFeeder/server.py index bb609f7..2a0eac4 100644 --- a/LocalFeeder/server.py +++ b/LocalFeeder/server.py @@ -1,19 +1,21 @@ +from fastapi import FastAPI, BackgroundTasks, UploadFile, Request +from fastapi.exceptions import HTTPException +from fastapi.responses import JSONResponse +from sender_cosim import run_simulator +import traceback import asyncio -import json import logging -import os +import zipfile +import uvicorn import socket -import sys +import json import time -import traceback -import zipfile +import sys +import os -import uvicorn -from fastapi import BackgroundTasks, FastAPI, Request, UploadFile -from fastapi.exceptions import HTTPException -from fastapi.responses import JSONResponse -from oedisi.types.common import BrokerConfig, HeathCheck, ServerReply -from sender_cosim import run_simulator +from oedisi.componentframework.system_configuration import ComponentStruct +from oedisi.types.common import ServerReply, HeathCheck, DefaultFileNames +from oedisi.types.common import BrokerConfig REQUEST_TIMEOUT_SEC = 1200 @@ -138,6 +140,21 @@ async def run_feeder( HTTPException(500, str(err)) +@app.post("/configure/") +async def configure(component_struct:ComponentStruct): + component = component_struct.component + params = component.parameters + params["name"] = component.name + links = {} + for link in component_struct.links: + links[link.target_port] = f"{link.source}/{link.source_port}" + json.dump(links , open(DefaultFileNames.INPUT_MAPPING.value, "w")) + json.dump(params , open(DefaultFileNames.STATIC_INPUTS.value, "w")) + response = ServerReply( + detail = f"Sucessfully updated configuration files." + ).dict() + return JSONResponse(response, 200) + if __name__ == "__main__": port = int(sys.argv[2]) - uvicorn.run(app, host="0.0.0.0", port=port) + uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/broker/server.py b/broker/server.py index e0509f6..2709f2e 100644 --- a/broker/server.py +++ b/broker/server.py @@ -1,21 +1,22 @@ -import logging -import os -import shutil -import socket -import sys -import time -import traceback -import zipfile - -import grequests +from fastapi import FastAPI, BackgroundTasks, UploadFile +from fastapi.responses import FileResponse, JSONResponse +from fastapi.exceptions import HTTPException import helics as h +import grequests +import traceback import requests +import zipfile import uvicorn +import logging +import socket +import shutil +import time import yaml -from fastapi import BackgroundTasks, FastAPI, UploadFile -from fastapi.exceptions import HTTPException -from fastapi.responses import FileResponse, JSONResponse -from oedisi.types.common import HeathCheck, ServerReply +import sys +import os + +from oedisi.componentframework.system_configuration import WiringDiagram, ComponentStruct +from oedisi.types.common import ServerReply, HeathCheck app = FastAPI() @@ -176,7 +177,23 @@ async def run_feeder(background_tasks: BackgroundTasks): err = traceback.format_exc() raise HTTPException(status_code=404, detail=str(err)) - + +@app.post("/configure/") +async def configure(wiring_diagram:WiringDiagram): + for component in wiring_diagram.components: + component_model = ComponentStruct( + component = component, + links = [] + ) + for link in wiring_diagram.links: + if link.target == component.name: + component_model.links.append(link) + + url = f'http://{component.host}:{component.container_port}/configure/' + logging.info(f"making post request to: {url}") + r = requests.post(url, json=component_model.dict()) + assert r.status_code==200, f"POST request to update configuration failed for url - {url}" + if __name__ == "__main__": port = int(sys.argv[2]) - uvicorn.run(app, host="0.0.0.0", port=port) + uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/measuring_federate/server.py b/measuring_federate/server.py index 10c4d63..015156e 100644 --- a/measuring_federate/server.py +++ b/measuring_federate/server.py @@ -1,21 +1,20 @@ from fastapi import FastAPI, BackgroundTasks, HTTPException -from oedisi.types.common import BrokerConfig from measuring_federate import run_simulator from fastapi.responses import JSONResponse +import traceback +import requests import uvicorn import socket -import requests import sys import json -import traceback +import os -import math -from oedisi.types.common import ServerReply, HeathCheck +from oedisi.componentframework.system_configuration import ComponentStruct +from oedisi.types.common import ServerReply, HeathCheck, DefaultFileNames +from oedisi.types.common import BrokerConfig app = FastAPI() - - @app.get("/") async def read_root(): hostname = socket.gethostname() @@ -37,6 +36,7 @@ async def run_model(broker_config:BrokerConfig, background_tasks: BackgroundTask print(url) reply = requests.get(url) sensor_data = reply.json() + print(sensor_data) with open("sensors.json", "w") as outfile: json.dump(sensor_data, outfile) @@ -49,7 +49,22 @@ async def run_model(broker_config:BrokerConfig, background_tasks: BackgroundTask err = traceback.format_exc() HTTPException(500,str(err)) +@app.post("/configure/") +async def configure(component_struct:ComponentStruct): + component = component_struct.component + params = component.parameters + params["name"] = component.name + links = {} + for link in component_struct.links: + links[link.target_port] = f"{link.source}/{link.source_port}" + json.dump(links , open(DefaultFileNames.INPUT_MAPPING.value, "w")) + json.dump(params , open(DefaultFileNames.STATIC_INPUTS.value, "w")) + response = ServerReply( + detail = f"Sucessfully updated configuration files." + ).dict() + return JSONResponse(response, 200) + if __name__ == "__main__": port = int(sys.argv[2]) - uvicorn.run(app, host="0.0.0.0", port=port) + uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/recorder/server.py b/recorder/server.py index 9481b85..e2ad116 100644 --- a/recorder/server.py +++ b/recorder/server.py @@ -1,15 +1,22 @@ + +import traceback import logging -import os import socket +import json import sys -import traceback +import os -import uvicorn -from fastapi import BackgroundTasks, FastAPI, HTTPException +from fastapi import FastAPI, BackgroundTasks, HTTPException from fastapi.responses import FileResponse, JSONResponse -from oedisi.types.common import BrokerConfig, HeathCheck, ServerReply +import uvicorn + +from oedisi.componentframework.system_configuration import ComponentStruct +from oedisi.types.common import ServerReply, HeathCheck, DefaultFileNames +from oedisi.types.common import BrokerConfig + from record_subscription import run_simulator + app = FastAPI() @@ -51,6 +58,21 @@ async def run_model(broker_config: BrokerConfig, background_tasks: BackgroundTas HTTPException(500, str(err)) +@app.post("/configure/") +async def configure(component_struct:ComponentStruct): + component = component_struct.component + params = component.parameters + params["name"] = component.name + links = {} + for link in component_struct.links: + links[link.target_port] = f"{link.source}/{link.source_port}" + json.dump(links , open(DefaultFileNames.INPUT_MAPPING.value, "w")) + json.dump(params , open(DefaultFileNames.STATIC_INPUTS.value, "w")) + response = ServerReply( + detail = f"Sucessfully updated configuration files." + ).dict() + return JSONResponse(response, 200) + if __name__ == "__main__": port = int(sys.argv[2]) - uvicorn.run(app, host="0.0.0.0", port=port) + uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/wls_federate/server.py b/wls_federate/server.py index 948ee8f..320f927 100644 --- a/wls_federate/server.py +++ b/wls_federate/server.py @@ -1,13 +1,16 @@ -from oedisi.types.common import BrokerConfig -from state_estimator_federate import run_simulator from fastapi import FastAPI, BackgroundTasks, HTTPException +from state_estimator_federate import run_simulator +from oedisi.types.common import BrokerConfig from fastapi.responses import JSONResponse import traceback import uvicorn import socket +import json import sys +import os -from oedisi.types.common import ServerReply, HeathCheck +from oedisi.componentframework.system_configuration import ComponentStruct +from oedisi.types.common import ServerReply, HeathCheck, DefaultFileNames app = FastAPI() @@ -37,6 +40,21 @@ async def run_model(broker_config: BrokerConfig, background_tasks: BackgroundTas HTTPException(500, str(err)) +@app.post("/configure/") +async def configure(component_struct:ComponentStruct): + component = component_struct.component + params = component.parameters + params["name"] = component.name + links = {} + for link in component_struct.links: + links[link.target_port] = f"{link.source}/{link.source_port}" + json.dump(links , open(DefaultFileNames.INPUT_MAPPING.value, "w")) + json.dump(params , open(DefaultFileNames.STATIC_INPUTS.value, "w")) + response = ServerReply( + detail = f"Sucessfully updated configuration files." + ).dict() + return JSONResponse(response, 200) + if __name__ == "__main__": port = int(sys.argv[2]) - uvicorn.run(app, host="0.0.0.0", port=port) + uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) From 5ae3ae0b6596f27475e3a33829516b049b1bd412 Mon Sep 17 00:00:00 2001 From: Latif Date: Thu, 29 Feb 2024 16:05:25 -0700 Subject: [PATCH 02/13] docker compose working --- LocalFeeder/requirements.txt | 2 +- LocalFeeder/server.py | 10 +- broker/__pycache__/server.cpython-312.pyc | Bin 0 -> 13183 bytes broker/docker-compose.yml | 19 ++ broker/requirements.txt | 2 +- broker/server.py | 71 +++++--- measuring_federate/requirements.txt | 2 +- measuring_federate/server.py | 36 ++-- recorder/requirements.txt | 2 +- recorder/server.py | 6 +- system.json | 203 ++++++++++++++++++++++ wls_federate/requirements.txt | 2 +- wls_federate/server.py | 4 +- 13 files changed, 312 insertions(+), 47 deletions(-) create mode 100644 broker/__pycache__/server.cpython-312.pyc create mode 100644 broker/docker-compose.yml create mode 100644 system.json diff --git a/LocalFeeder/requirements.txt b/LocalFeeder/requirements.txt index cd911b4..1256674 100644 --- a/LocalFeeder/requirements.txt +++ b/LocalFeeder/requirements.txt @@ -11,5 +11,5 @@ boto3 xarray fastapi uvicorn -oedisi==1.1.1 +git+https://github.com/openEDI/oedisi@al/config python-multipart \ No newline at end of file diff --git a/LocalFeeder/server.py b/LocalFeeder/server.py index 2a0eac4..a77cf97 100644 --- a/LocalFeeder/server.py +++ b/LocalFeeder/server.py @@ -51,7 +51,7 @@ def read_root(): return JSONResponse(response, 200) -@app.get("/sensor/") +@app.get("/sensor") async def sensor(): logging.info(os.getcwd()) sensor_path = os.path.join(base_path, "sensors", "sensors.json") @@ -63,7 +63,7 @@ async def sensor(): return data -@app.post("/profiles/") +@app.post("/profiles") async def upload_profiles(file: UploadFile): try: data = file.file.read() @@ -96,7 +96,7 @@ async def upload_profiles(file: UploadFile): ) -@app.post("/model/") +@app.post("/model") async def upload_model(file: UploadFile): try: data = file.file.read() @@ -125,7 +125,7 @@ async def upload_model(file: UploadFile): HTTPException(500, "Unknown error while uploading userdefined opendss model.") -@app.post("/run/") +@app.post("/run") async def run_feeder( broker_config: BrokerConfig, background_tasks: BackgroundTasks ): # :BrokerConfig @@ -140,7 +140,7 @@ async def run_feeder( HTTPException(500, str(err)) -@app.post("/configure/") +@app.post("/configure") async def configure(component_struct:ComponentStruct): component = component_struct.component params = component.parameters diff --git a/broker/__pycache__/server.cpython-312.pyc b/broker/__pycache__/server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d3cf7595fa9ddeda889bc0a17218640f50fcf76 GIT binary patch literal 13183 zcmeG?X>1(Vc{4jRdtc;oxl56fmLhemB$7HPiK1@t)Gb?bY;UBjS35&$S9>rsLrJ8U z3T+E8l?o757|=S7>rEOoRIMN^k{?33L0nWo>pul832-8(K)-Kh zXLh-!>@)?6qR1ondvD(R-u>QpzJGK$tOSI8|C#5c073jKerUm5Og!2`5yUNmC0H^+ zB*_q&G=vOEDnyZ(XGqXVCd4F-A*1q5B}_?k$gHI4ge7DJ96DhO*_7BGvMaG8x%VYfLOl z)`#ko4WR~-Fc2L1ZgDE1<*bRTjqlc`Sc)$ayld$a)>H>*DUzi_D_JuaU@crDYvooA z7+Kp@I@AR1*T(xxEomvOPu4zQU>#S<&}!Csl?bijmd8(MIHj0%y-TWJtotgV)eWr! z>{BHyEyY&p*wq{b?R&1$dY$!8sZ&!?$6Hos!&638tJm?BjcVglu>CrAT^V)@yA0;t z`jk=C>vbA91I%Q3*{Ir{QfGx;XC>!{I)Sn}o4Czv_dR$H}F%&x*c z)mjN{1x#1`e5qVZam!iU-^Qn$jOOxI?yP945d>7C)*F7I&ob43X-y;|McmT zJ(r{0kQhs+jsw2q`4}Hd4R*&OgM1_@ySmazsLG|pQzAbc6#>I`isLVG{IlFpVnnv~ za}n`;*Lf~_LAIajdG>3Ex_ZLL4;<@}>GNqplixj!$tS zCxp4wMcI%RaGn=qd^#o5ahPt9;hoR`??T{4fbGfDFrN?@NChxF+27T9<~f1mg)@;v zM2rob=}t$7ld#~znZB6VKYZ?tFc@Pai5Bj1BsrAe&Ya`Z7dZY5RDoS+9U74>=Z0em zHVn=3_0a4x_zB+tFi!l+E2$m`w=b<=R&PeYwC>J&gyNnIj>VGn0Ov`(KZA_ z4~ywA78r%$fq~d%*)jkZCKX9?f^624cnWd1cwAiBIS@;+VZD%F4Vg{w6E*`FC+6MN zx2!j<*2kb>4$Ru(7&n{Ch5|_xJ?f7Uln~uW0Y>6Vl6u8!rWI`t4yx~2KMd36#(i|HT zVqx|E%4Q%Pm(u(N;fXUkEQsl3OEE*X$A&Jpha)V@bAlix&?w{)=l-!S9x!(xVa){EL5$XFn&fs=EpAYWL$DJ zj`z>gmWg#&Ps};IH_lu;lWi+F){pnjxoam!CHJa)(^kpdKK%{J-93J2-c>!hJ-hbi zzT9TXvwGIELGo;v>MD3PO;<^ttp!*6c>l+q`iBI?G)|bHy~#nzu_8MvIo9Vl?2;V2 z?|wsa9Gx&=GacDJ$B>IxrV@mWAfYw z=DGzFw7Fm<>@|~TbBCrb=WRRk^p4N!VIs;Dp9pvlZq#-esP_#_m(9f2!1tnyj^qWe z@1ySn_$qp-$zWeeM8x$GL(o7uto|c1v4^uB$ToV zQ*dBD`2`F=XoT~HS3#6*gPf=%If@_WMz9WV2UMA6V^NVuq62h#A4L3q%r?WpSL77| z7X?n3db}*k@#52Ik?({2PWTB20gMw5T;5q%Bb=Cms|gN`vu4)0N`hn7QE+Y??|a~? z%XLgYKjYdn>pCd84i;S9dL>Lflr70iW9VfDjXL+_Z`i zN9m>_ux70%!7f&5g{%dLe34<1gow0E2}`%j+B95XZl;VW?$z*1amJ|iqiAT0n40L* z@G^9UVeNyYCP|N(MO;uN#F3v@1lEx;zeT*Q3x?3P^D9T|(ncF2$1LJb>F8xi0LO8oH&-8u@SXcF%3leR;ti;+Z( z4d87Kh$BN>Yv4pGF%kd_B~J)McrFl$q5{ORt+L_LIoYgHfhWt7kqf}d0+9gEeRG%- z#DJI%fS4U<3Gn^UschiS-6my5p&5`cvYZ%+B|dor`W~%&E)_Y4y+Rd5Aq!~j-Ub~? z5F_HS5RRr{KF>i@pFD;-x9b$1DKm-mC79+`oL4&nr864IAK#4M0R(t~d0cauxx~jr zP7(e=&`l2x!k}e3mKsPSDOE-;pe6=v--h_K$Xyos9;}a4jeh|%ExH;gTSPt*<<5a# zDO(59d@>@2xyw}`G}~t?3w7BbNRA6-r4i!p11aW;95WDU7j73ylZl!l6OmfYnSBhzV{8udwS9^ z?^~Ab$n{C}8>gO^>bFftr23B8`n^*9-n&AfzU$rrslK=1>zkzKeD$|RZjRi#a`Q^T zw^scgyE!)NYms~{Qv)-;ZSysabN>4L^5c&wgSYmvk*HmkZGNZa&6cT}g0~d}fT~r~ zjdL~LTYWeCvfXbS0|BRM6`=V7IYX{{s&i^%p?1e4HShM_vfs4lSMI#qbl00-zHi38 z|KqxqKd8yEKTLivIeog&*qQh5nq=l{8g3oCc`Vm7_1sL&)=6@{e$7%j@*1?6cr(o}lKP^;6VOyY5my>7Q}#$(#3lzQ6#|?;ah4jd_*q-9cVm z+q0XvbI=CxXFIBUHZkv84*KEaej}FNU**K~CZ=bn<^DDkrgvL<+o_*3>wDX%pSLlX zZl?fq)YO)Q>yT*sBisEW+eL%dAKC8p{|juF2lC|80tAyQGTP`QG? z3kn0|Xi>M6f!zqKUS(=WnLTVNMT)g# zn8j*`sH;|Sv>sQ445Nw8W5zP|!~j-s>)VFK>XWD|PjR${R)mZ(?yty#&x}b^2S8av zyRs7Es3}(j)~=1@i)z^Q0!^?EP`OrWmEr*{Rf;oa)~WRd>Yhbx(#o{Ghuwy}F>A)M zloFJ&in^i|U!&D6#dmb|3~Y9*X;9oNZJ?ma2Q_?9(9EnG)Gj=Y5WZ;lnAYLhGMOXHoG~sP8AJ zXph>87Y#KcnY@G!AG8FbT``Q>kW4`VIFo2Jge$6rY)yidGNP2K6x!7bDhcm0gx%M`>1M)Q8+&=#uBnEnjRWKohf}TE>nEu5-21w zFe5QSg5!o{6O0KyKF&uGpe_`&D_RME z9y1o}C8|22C?V$bu&9{sRYQLmXD&0)Ie!7#5zykHX(<@1nu=RnjBs&* z`3b-q!wTb|s(?YtzxpkgM0>~kCnA$olLzL|^k`#Nfa%fZ&HCP1{^s(xR(@bxH&<1c zT`N_s%5_Rrt@*ZnQq}%@E2OHE69?x#wUd`~Cds#M$_K8S=|;)dKI`k0e4Tf<7JU2e zk&^FV!P7O-{h+!&doV}mMstZm_11~5dAsMv(Q8NZpw>jEpUcd^gL!VMu`Wb~a7R(gWN6x$Ib9K`%{$ksV>zTayna{k? z;iovZC!bjrEKpAllb=7fE2;{KHBdXrxA)9Ad-LYr&lkXY+($kUHo;E(Y|Zu_6Z8Jw z%{?{L-|lqu1c>`4rl-nszlP~qZn__^^z5SU2OT}zsr%a*NRQguK!;AoQW23`%$U&b zhNR%pRRD{veoOhYG?E2mQ@7ZmvnEEelt$!`q=nBMYOv<&tb+w-12F)H4G#yur2BU#~E)k0g0o7z55Qf3eAPfvAz`_wtfD?k{4?qX%J$6^z;{dN* zWJV^>%hl(VcDk_6(bzdY!jGtyfzst7)%yUan=xwGec(uNHacc=rOR)@V%+MM?0neCzFqUe)#M$8pmD)v8R1r#;AOn19 zo2)5gtQhf^;+JHP(h8Nh{2-cS$#_-8aNtur{J^ils&odp(3lzcxktl9cU&+O;&Ut0JR-(mVoE(Nn|@C#+jJt^Gc9^F843;7FU`Kl}9hv#YA>qlQXn)hs( z?wsD3ui7_5@1J)#XB~}_qcOL7s(LC^aCD6K7V#!)XK4R?wP&_^gH*j?suyI9@gsU0 zt8*7}XY*BCXXy4~-W#6DvxTar8M=AS=^pQ!Yu-HDyhCc21?%3(ih3H=12H?APphka(J=y?O7B8FR;ik!Wa~t6Mea55WJY7N*KN!7SJb zkUX5%oNp-B*<5s*nelD^(7a>82dzK$6O8lqW3L>WTsuQAo449#txb}(Dc3t=-7s&q z->_b@PHw*LcuWyCyH@m}b;ARinWg;_?T1Pq(^l0j$h5(-x5;j_hpX?b!XiGjTG~wc zJJ=7+I|!aFJhc^}^th>vnCQfqCGY5-r=@&)+Qo(kA3AG*Cr~6v7qmvw6zM`|LyDiq zAUd>R#L@m<5%gB6GCvJdMz;0mAQf_n;fkRomneQ3)A3OsK15q2jlg3DaIdI#A9Qz= znCd`W1 zlDnL5*fiZ#aBa_AdISSGlqGtO?F;V-sHhECFN1xqJxLh$!qX?WCdQU9y~a9kWm?cBmWa{<3+O1 zAW?b>0PbpHsb>UGZwJ)FhgjHGMF1%PH=`opQ%P`VF9*vEZbd~{%FSH<-st*H;KTZH zg&vkTu#mfiO@uX+*<0w+@r~n)i;4i3rc#D0@&2XkAvDYe72JxTU7b7n^#k`hv!tDb zQJ#>J*Y6*Omr}*vi>eqG8eB{eVv~mV(Ej zkb_Tt;8>{!1AKd_T*lGuCr_L@y~xoy1kOAzpjA-W`+#Z;0Iy)t5jwi{RGHQ>tkhwP zm5MG_M2;j9moFBY>wT~I9!8+kuFcW}Mvrej62d*{C z)`{*pm*>XI*Iv%X-buZgD!4YzxmV7*S4-~Iv+j+Od*jsU58Ye9!DiX|*i5+mS<5@l zH=R?K*>(G+b^Gt_y4O;0o?MzqKCsu^ICkw=wkdaFwz)%U?kF^O&e(TNkaI5gjj?NE z*=WJlm^VlAtG}K<6V8AArF>Il#vECw$B{p7Ang9{+VUG-n(;?w{R5JJpx{5Bk0%SZ zRGv=#RzUgv)dNg7MZRY~XoKkenl^y4ISdcVVySSL9|nw3)23GCPo__vc=mMAPJ`=0B@f;e4W<{ zI3Oj0qh7r>s*;PGS2?he$FtFSD^T!tqOWJNsI zvJuXD^b)IDx%MF1%Lw`qL=YgW;?b?4SeXq^`+%1v9F}RgI~Ua_$CFTe zn^1%p#ccJ&yc>@DaDv;*{|t&yG!gzDAXrUE^4CO%M05cB6|wOnqWUAkt^B(_BAnX4 z<0At8ZOXs(Bf|1;#EZWscKnOc@mg%0`gqI!dk6pS@Oy`)EypJh-a35qaQ4avHLdy6 zFXV};1*3^{O>SKvAj+OnqN(Ob__a_?1cF)qosl<3-g0(`0VXLjq&$u1?KbSVhqGiO~c_k1XxDh(>qlNWGD&5Fx0|NU{WyL*ez#U?-CtOL;Ft8l1%{xPDO>$t D&j Date: Thu, 29 Feb 2024 16:42:35 -0700 Subject: [PATCH 03/13] fixed host/hostname bug --- broker/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/broker/server.py b/broker/server.py index 08ec9c0..8bfa177 100644 --- a/broker/server.py +++ b/broker/server.py @@ -75,7 +75,7 @@ async def upload_profiles(file: UploadFile): services, _, _, _ = read_settings() for service in services: if "feeder" in service.lower(): - ip = services[service]["host"] + ip = services[service]["hostname"] port = int(services[service]["ports"][0].split(":")[0]) data = file.file.read() if not file.filename.endswith(".zip"): @@ -104,7 +104,7 @@ async def upload_model(file: UploadFile): services, _, _, _ = read_settings() for service in services: if "feeder" in service.lower(): - ip = services[service]["host"] + ip = services[service]["hostname"] port = int(services[service]["ports"][0].split(":")[0]) data = file.file.read() if not file.filename.endswith(".zip"): From 79c391f30a43d9abd6b4971ca8ddcb159b37deb4 Mon Sep 17 00:00:00 2001 From: Latif Date: Tue, 19 Mar 2024 15:04:03 -0600 Subject: [PATCH 04/13] minor edits --- broker/__pycache__/server.cpython-312.pyc | Bin 13183 -> 0 bytes broker/server.py | 18 +++++++++--------- measuring_federate/server.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) delete mode 100644 broker/__pycache__/server.cpython-312.pyc diff --git a/broker/__pycache__/server.cpython-312.pyc b/broker/__pycache__/server.cpython-312.pyc deleted file mode 100644 index 8d3cf7595fa9ddeda889bc0a17218640f50fcf76..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13183 zcmeG?X>1(Vc{4jRdtc;oxl56fmLhemB$7HPiK1@t)Gb?bY;UBjS35&$S9>rsLrJ8U z3T+E8l?o757|=S7>rEOoRIMN^k{?33L0nWo>pul832-8(K)-Kh zXLh-!>@)?6qR1ondvD(R-u>QpzJGK$tOSI8|C#5c073jKerUm5Og!2`5yUNmC0H^+ zB*_q&G=vOEDnyZ(XGqXVCd4F-A*1q5B}_?k$gHI4ge7DJ96DhO*_7BGvMaG8x%VYfLOl z)`#ko4WR~-Fc2L1ZgDE1<*bRTjqlc`Sc)$ayld$a)>H>*DUzi_D_JuaU@crDYvooA z7+Kp@I@AR1*T(xxEomvOPu4zQU>#S<&}!Csl?bijmd8(MIHj0%y-TWJtotgV)eWr! z>{BHyEyY&p*wq{b?R&1$dY$!8sZ&!?$6Hos!&638tJm?BjcVglu>CrAT^V)@yA0;t z`jk=C>vbA91I%Q3*{Ir{QfGx;XC>!{I)Sn}o4Czv_dR$H}F%&x*c z)mjN{1x#1`e5qVZam!iU-^Qn$jOOxI?yP945d>7C)*F7I&ob43X-y;|McmT zJ(r{0kQhs+jsw2q`4}Hd4R*&OgM1_@ySmazsLG|pQzAbc6#>I`isLVG{IlFpVnnv~ za}n`;*Lf~_LAIajdG>3Ex_ZLL4;<@}>GNqplixj!$tS zCxp4wMcI%RaGn=qd^#o5ahPt9;hoR`??T{4fbGfDFrN?@NChxF+27T9<~f1mg)@;v zM2rob=}t$7ld#~znZB6VKYZ?tFc@Pai5Bj1BsrAe&Ya`Z7dZY5RDoS+9U74>=Z0em zHVn=3_0a4x_zB+tFi!l+E2$m`w=b<=R&PeYwC>J&gyNnIj>VGn0Ov`(KZA_ z4~ywA78r%$fq~d%*)jkZCKX9?f^624cnWd1cwAiBIS@;+VZD%F4Vg{w6E*`FC+6MN zx2!j<*2kb>4$Ru(7&n{Ch5|_xJ?f7Uln~uW0Y>6Vl6u8!rWI`t4yx~2KMd36#(i|HT zVqx|E%4Q%Pm(u(N;fXUkEQsl3OEE*X$A&Jpha)V@bAlix&?w{)=l-!S9x!(xVa){EL5$XFn&fs=EpAYWL$DJ zj`z>gmWg#&Ps};IH_lu;lWi+F){pnjxoam!CHJa)(^kpdKK%{J-93J2-c>!hJ-hbi zzT9TXvwGIELGo;v>MD3PO;<^ttp!*6c>l+q`iBI?G)|bHy~#nzu_8MvIo9Vl?2;V2 z?|wsa9Gx&=GacDJ$B>IxrV@mWAfYw z=DGzFw7Fm<>@|~TbBCrb=WRRk^p4N!VIs;Dp9pvlZq#-esP_#_m(9f2!1tnyj^qWe z@1ySn_$qp-$zWeeM8x$GL(o7uto|c1v4^uB$ToV zQ*dBD`2`F=XoT~HS3#6*gPf=%If@_WMz9WV2UMA6V^NVuq62h#A4L3q%r?WpSL77| z7X?n3db}*k@#52Ik?({2PWTB20gMw5T;5q%Bb=Cms|gN`vu4)0N`hn7QE+Y??|a~? z%XLgYKjYdn>pCd84i;S9dL>Lflr70iW9VfDjXL+_Z`i zN9m>_ux70%!7f&5g{%dLe34<1gow0E2}`%j+B95XZl;VW?$z*1amJ|iqiAT0n40L* z@G^9UVeNyYCP|N(MO;uN#F3v@1lEx;zeT*Q3x?3P^D9T|(ncF2$1LJb>F8xi0LO8oH&-8u@SXcF%3leR;ti;+Z( z4d87Kh$BN>Yv4pGF%kd_B~J)McrFl$q5{ORt+L_LIoYgHfhWt7kqf}d0+9gEeRG%- z#DJI%fS4U<3Gn^UschiS-6my5p&5`cvYZ%+B|dor`W~%&E)_Y4y+Rd5Aq!~j-Ub~? z5F_HS5RRr{KF>i@pFD;-x9b$1DKm-mC79+`oL4&nr864IAK#4M0R(t~d0cauxx~jr zP7(e=&`l2x!k}e3mKsPSDOE-;pe6=v--h_K$Xyos9;}a4jeh|%ExH;gTSPt*<<5a# zDO(59d@>@2xyw}`G}~t?3w7BbNRA6-r4i!p11aW;95WDU7j73ylZl!l6OmfYnSBhzV{8udwS9^ z?^~Ab$n{C}8>gO^>bFftr23B8`n^*9-n&AfzU$rrslK=1>zkzKeD$|RZjRi#a`Q^T zw^scgyE!)NYms~{Qv)-;ZSysabN>4L^5c&wgSYmvk*HmkZGNZa&6cT}g0~d}fT~r~ zjdL~LTYWeCvfXbS0|BRM6`=V7IYX{{s&i^%p?1e4HShM_vfs4lSMI#qbl00-zHi38 z|KqxqKd8yEKTLivIeog&*qQh5nq=l{8g3oCc`Vm7_1sL&)=6@{e$7%j@*1?6cr(o}lKP^;6VOyY5my>7Q}#$(#3lzQ6#|?;ah4jd_*q-9cVm z+q0XvbI=CxXFIBUHZkv84*KEaej}FNU**K~CZ=bn<^DDkrgvL<+o_*3>wDX%pSLlX zZl?fq)YO)Q>yT*sBisEW+eL%dAKC8p{|juF2lC|80tAyQGTP`QG? z3kn0|Xi>M6f!zqKUS(=WnLTVNMT)g# zn8j*`sH;|Sv>sQ445Nw8W5zP|!~j-s>)VFK>XWD|PjR${R)mZ(?yty#&x}b^2S8av zyRs7Es3}(j)~=1@i)z^Q0!^?EP`OrWmEr*{Rf;oa)~WRd>Yhbx(#o{Ghuwy}F>A)M zloFJ&in^i|U!&D6#dmb|3~Y9*X;9oNZJ?ma2Q_?9(9EnG)Gj=Y5WZ;lnAYLhGMOXHoG~sP8AJ zXph>87Y#KcnY@G!AG8FbT``Q>kW4`VIFo2Jge$6rY)yidGNP2K6x!7bDhcm0gx%M`>1M)Q8+&=#uBnEnjRWKohf}TE>nEu5-21w zFe5QSg5!o{6O0KyKF&uGpe_`&D_RME z9y1o}C8|22C?V$bu&9{sRYQLmXD&0)Ie!7#5zykHX(<@1nu=RnjBs&* z`3b-q!wTb|s(?YtzxpkgM0>~kCnA$olLzL|^k`#Nfa%fZ&HCP1{^s(xR(@bxH&<1c zT`N_s%5_Rrt@*ZnQq}%@E2OHE69?x#wUd`~Cds#M$_K8S=|;)dKI`k0e4Tf<7JU2e zk&^FV!P7O-{h+!&doV}mMstZm_11~5dAsMv(Q8NZpw>jEpUcd^gL!VMu`Wb~a7R(gWN6x$Ib9K`%{$ksV>zTayna{k? z;iovZC!bjrEKpAllb=7fE2;{KHBdXrxA)9Ad-LYr&lkXY+($kUHo;E(Y|Zu_6Z8Jw z%{?{L-|lqu1c>`4rl-nszlP~qZn__^^z5SU2OT}zsr%a*NRQguK!;AoQW23`%$U&b zhNR%pRRD{veoOhYG?E2mQ@7ZmvnEEelt$!`q=nBMYOv<&tb+w-12F)H4G#yur2BU#~E)k0g0o7z55Qf3eAPfvAz`_wtfD?k{4?qX%J$6^z;{dN* zWJV^>%hl(VcDk_6(bzdY!jGtyfzst7)%yUan=xwGec(uNHacc=rOR)@V%+MM?0neCzFqUe)#M$8pmD)v8R1r#;AOn19 zo2)5gtQhf^;+JHP(h8Nh{2-cS$#_-8aNtur{J^ils&odp(3lzcxktl9cU&+O;&Ut0JR-(mVoE(Nn|@C#+jJt^Gc9^F843;7FU`Kl}9hv#YA>qlQXn)hs( z?wsD3ui7_5@1J)#XB~}_qcOL7s(LC^aCD6K7V#!)XK4R?wP&_^gH*j?suyI9@gsU0 zt8*7}XY*BCXXy4~-W#6DvxTar8M=AS=^pQ!Yu-HDyhCc21?%3(ih3H=12H?APphka(J=y?O7B8FR;ik!Wa~t6Mea55WJY7N*KN!7SJb zkUX5%oNp-B*<5s*nelD^(7a>82dzK$6O8lqW3L>WTsuQAo449#txb}(Dc3t=-7s&q z->_b@PHw*LcuWyCyH@m}b;ARinWg;_?T1Pq(^l0j$h5(-x5;j_hpX?b!XiGjTG~wc zJJ=7+I|!aFJhc^}^th>vnCQfqCGY5-r=@&)+Qo(kA3AG*Cr~6v7qmvw6zM`|LyDiq zAUd>R#L@m<5%gB6GCvJdMz;0mAQf_n;fkRomneQ3)A3OsK15q2jlg3DaIdI#A9Qz= znCd`W1 zlDnL5*fiZ#aBa_AdISSGlqGtO?F;V-sHhECFN1xqJxLh$!qX?WCdQU9y~a9kWm?cBmWa{<3+O1 zAW?b>0PbpHsb>UGZwJ)FhgjHGMF1%PH=`opQ%P`VF9*vEZbd~{%FSH<-st*H;KTZH zg&vkTu#mfiO@uX+*<0w+@r~n)i;4i3rc#D0@&2XkAvDYe72JxTU7b7n^#k`hv!tDb zQJ#>J*Y6*Omr}*vi>eqG8eB{eVv~mV(Ej zkb_Tt;8>{!1AKd_T*lGuCr_L@y~xoy1kOAzpjA-W`+#Z;0Iy)t5jwi{RGHQ>tkhwP zm5MG_M2;j9moFBY>wT~I9!8+kuFcW}Mvrej62d*{C z)`{*pm*>XI*Iv%X-buZgD!4YzxmV7*S4-~Iv+j+Od*jsU58Ye9!DiX|*i5+mS<5@l zH=R?K*>(G+b^Gt_y4O;0o?MzqKCsu^ICkw=wkdaFwz)%U?kF^O&e(TNkaI5gjj?NE z*=WJlm^VlAtG}K<6V8AArF>Il#vECw$B{p7Ang9{+VUG-n(;?w{R5JJpx{5Bk0%SZ zRGv=#RzUgv)dNg7MZRY~XoKkenl^y4ISdcVVySSL9|nw3)23GCPo__vc=mMAPJ`=0B@f;e4W<{ zI3Oj0qh7r>s*;PGS2?he$FtFSD^T!tqOWJNsI zvJuXD^b)IDx%MF1%Lw`qL=YgW;?b?4SeXq^`+%1v9F}RgI~Ua_$CFTe zn^1%p#ccJ&yc>@DaDv;*{|t&yG!gzDAXrUE^4CO%M05cB6|wOnqWUAkt^B(_BAnX4 z<0At8ZOXs(Bf|1;#EZWscKnOc@mg%0`gqI!dk6pS@Oy`)EypJh-a35qaQ4avHLdy6 zFXV};1*3^{O>SKvAj+OnqN(Ob__a_?1cF)qosl<3-g0(`0VXLjq&$u1?KbSVhqGiO~c_k1XxDh(>qlNWGD&5Fx0|NU{WyL*ez#U?-CtOL;Ft8l1%{xPDO>$t D&j Date: Tue, 26 Mar 2024 12:47:59 -0600 Subject: [PATCH 05/13] minimal edits to support changes to multicontainer implementation --- LocalFeeder/server.py | 1 - broker/Dockerfile | 2 +- broker/server.py | 28 ++++++++++++++++------------ measuring_federate/server.py | 1 - recorder/server.py | 1 - wls_federate/server.py | 1 - 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/LocalFeeder/server.py b/LocalFeeder/server.py index a77cf97..85032fc 100644 --- a/LocalFeeder/server.py +++ b/LocalFeeder/server.py @@ -156,5 +156,4 @@ async def configure(component_struct:ComponentStruct): return JSONResponse(response, 200) if __name__ == "__main__": - port = int(sys.argv[2]) uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/broker/Dockerfile b/broker/Dockerfile index b6ce2a9..0c009d8 100644 --- a/broker/Dockerfile +++ b/broker/Dockerfile @@ -12,4 +12,4 @@ RUN pip install -r requirements.txt EXPOSE 8766/tcp -CMD ["python", "server.py", "10.5.0.2", "8766"] +CMD ["python", "server.py"] diff --git a/broker/server.py b/broker/server.py index eafa0c2..f5c2340 100644 --- a/broker/server.py +++ b/broker/server.py @@ -19,6 +19,10 @@ from oedisi.componentframework.system_configuration import WiringDiagram, ComponentStruct from oedisi.types.common import ServerReply, HeathCheck + +logger = logging.getLogger('uvicorn.error') + + app = FastAPI() is_kubernetes_env = os.environ['KUBERNETES_SERVICE_NAME'] if 'KUBERNETES_SERVICE_NAME' in os.environ else None @@ -86,7 +90,7 @@ async def upload_profiles(file: UploadFile): f.write(data) url = build_url(ip, port, ["profiles"]) - logging.info(f"making a request to url - {url}") + logger.info(f"making a request to url - {url}") files = {"file": open(file.filename, "rb")} r = requests.post(url, files=files) @@ -115,7 +119,7 @@ async def upload_model(file: UploadFile): f.write(data) url = build_url(ip, port, ["model"]) - logging.info(f"making a request to url - {url}") + logger.info(f"making a request to url - {url}") files = {"file": open(file.filename, "rb")} r = requests.post(url, files=files) @@ -135,10 +139,10 @@ def download_results(): port = int(services[service]["ports"][0].split(":")[0]) url = build_url(host, port, ["download"]) - logging.info(f"making a request to url - {url}") + logger.info(f"making a request to url - {url}") response = requests.get(url) - logging.info(f"Response from {service} has {len(response.content)} bytes") + logger.info(f"Response from {service} has {len(response.content)} bytes") with open(f"{service}.feather", "wb") as out_file: out_file.write(response.content) @@ -164,19 +168,20 @@ def terminate_simulation(): def run_simulation(): services, component_map, broker_ip, api_port = read_settings() - logging.info(f"{broker_ip}, {api_port}") + print(services) + logger.info(f"{broker_ip}, {api_port}") initstring = f"-f {len(component_map)} --name=mainbroker --loglevel=trace --local_interface={broker_ip} --localport=23404" - logging.info(f"Broker initaialization string: {initstring}") + logger.info(f"Broker initaialization string: {initstring}") broker = h.helicsCreateBroker("zmq", "", initstring) - logging.info(broker) + logger.info(broker) isconnected = h.helicsBrokerIsConnected(broker) - logging.info(f"Broker connected: {isconnected}") - logging.info(str(component_map)) + logger.info(f"Broker connected: {isconnected}") + logger.info(str(component_map)) replies = [] for service_ip, service_port in component_map.items(): url = build_url(service_ip, service_port, ["run"]) - logging.info(f"making a request to url - {url}") + logger.info(f"making a request to url - {url}") myobj = { "broker_port": 23404, @@ -217,12 +222,11 @@ async def configure(wiring_diagram:WiringDiagram): component_model.links.append(link) url = build_url(component.host, component.container_port, ["configure"]) - logging.info(f"making a request to url - {url}") + logger.info(f"making a request to url - {url}") r = requests.post(url, json=component_model.dict()) assert r.status_code==200, f"POST request to update configuration failed for url - {url}" return JSONResponse(ServerReply(detail="Sucessfully updated config files for all containers").dict(), 200) if __name__ == "__main__": - port = int(sys.argv[2]) uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/measuring_federate/server.py b/measuring_federate/server.py index b9e88d5..7ea47c3 100644 --- a/measuring_federate/server.py +++ b/measuring_federate/server.py @@ -79,6 +79,5 @@ async def configure(component_struct:ComponentStruct): return JSONResponse(response, 200) if __name__ == "__main__": - port = int(sys.argv[2]) uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/recorder/server.py b/recorder/server.py index dba2df7..8e5249d 100644 --- a/recorder/server.py +++ b/recorder/server.py @@ -74,5 +74,4 @@ async def configure(component_struct:ComponentStruct): return JSONResponse(response, 200) if __name__ == "__main__": - port = int(sys.argv[2]) uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) diff --git a/wls_federate/server.py b/wls_federate/server.py index d7cf70b..c907802 100644 --- a/wls_federate/server.py +++ b/wls_federate/server.py @@ -56,5 +56,4 @@ async def configure(component_struct:ComponentStruct): return JSONResponse(response, 200) if __name__ == "__main__": - port = int(sys.argv[2]) uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) From 17838ba9fd52937e3d3f9ef046292494ee57d429 Mon Sep 17 00:00:00 2001 From: Latif Date: Thu, 28 Mar 2024 14:41:21 -0600 Subject: [PATCH 06/13] moving system.json to scenarios folder --- system.json => scenarios/system.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename system.json => scenarios/system.json (100%) diff --git a/system.json b/scenarios/system.json similarity index 100% rename from system.json rename to scenarios/system.json From 3e6b426eeb89e74ae36398282c78f8cf95585792 Mon Sep 17 00:00:00 2001 From: Latif Date: Fri, 29 Mar 2024 10:47:38 -0600 Subject: [PATCH 07/13] example updates --- broker/server.py | 102 +++++++++++++++++++---------------- measuring_federate/server.py | 4 +- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/broker/server.py b/broker/server.py index f5c2340..2a8629c 100644 --- a/broker/server.py +++ b/broker/server.py @@ -9,18 +9,19 @@ import uvicorn import logging import socket -import shutil import time import yaml import json -import sys import os +import json + from oedisi.componentframework.system_configuration import WiringDiagram, ComponentStruct from oedisi.types.common import ServerReply, HeathCheck -logger = logging.getLogger('uvicorn.error') +#logger = logging.getLogger('uvicorn.error') +logger = logging.getLogger(__name__) app = FastAPI() @@ -28,6 +29,14 @@ is_kubernetes_env = os.environ['KUBERNETES_SERVICE_NAME'] if 'KUBERNETES_SERVICE_NAME' in os.environ else None WIRING_DIAGRAM_FILENAME = "system.json" +WIRING_DIAGRAM : WiringDiagram | None = None + + + +# def test_function(json_file = r"C:/Users/alatif/Documents/GitHub/sgidal-example/scenarios/system.json"): +# global WIRING_DIAGRAM +# data = json.load(open(json_file, "r")) +# WIRING_DIAGRAM = WiringDiagram(**data) def build_url(host:str, port:int, enpoint:list): if is_kubernetes_env: @@ -44,23 +53,23 @@ def find_filenames(path_to_dir=os.getcwd(), suffix=".feather"): def read_settings(): - component_map = {} - with open("docker-compose.yml", "r") as stream: - config = yaml.safe_load(stream) - services = config["services"] - print(services) - broker = services.pop("oedisi_broker") - broker_host = broker["hostname"] - - broker_ip = socket.gethostbyname(broker_host) - api_port = int(broker["ports"][0].split(":")[0]) - - for service in services: - host = services[service]["hostname"] - port = int(services[service]["ports"][0].split(":")[0]) - component_map[host] = port + + broker_host = socket.gethostname() + broker_ip = socket.gethostbyname(broker_host) + api_port = 8766 #int(os.environ['PORT']) + + component_map = { + broker_host: api_port + } + if WIRING_DIAGRAM: + print(WIRING_DIAGRAM) + print("") + for component in WIRING_DIAGRAM.components: + component_map[component.host] = component.container_port + else: + logger.info("Use the '/configure' setpoint to setup up the WiringDiagram before making requests other enpoints") - return services, component_map, broker_ip, api_port + return component_map, broker_ip, api_port @app.get("/") @@ -72,15 +81,14 @@ def read_root(): return JSONResponse(response, 200) - @app.post("/profiles") async def upload_profiles(file: UploadFile): try: - services, _, _, _ = read_settings() - for service in services: - if "feeder" in service.lower(): - ip = services[service]["hostname"] - port = int(services[service]["ports"][0].split(":")[0]) + component_map, _, _ = read_settings() + for hostname in component_map: + if "feeder" in hostname(): + ip = hostname + port = component_map[hostname] data = file.file.read() if not file.filename.endswith(".zip"): HTTPException( @@ -101,15 +109,14 @@ async def upload_profiles(file: UploadFile): err = traceback.format_exc() raise HTTPException(status_code=500, detail=str(err)) - @app.post("/model") async def upload_model(file: UploadFile): try: - services, _, _, _ = read_settings() - for service in services: - if "feeder" in service.lower(): - ip = services[service]["hostname"] - port = int(services[service]["ports"][0].split(":")[0]) + component_map, _, _ = read_settings() + for hostname in component_map: + if "feeder" in hostname(): + ip = hostname + port = component_map[hostname] data = file.file.read() if not file.filename.endswith(".zip"): HTTPException( @@ -132,18 +139,19 @@ async def upload_model(file: UploadFile): @app.get("/results") def download_results(): - services, _, _, _ = read_settings() - for service in services: - if "recorder" in service.lower(): - host = services[service]["hostname"] - port = int(services[service]["ports"][0].split(":")[0]) + component_map, _, _ = read_settings() + + for hostname in component_map: + if "recorder" in hostname(): + host = hostname + port = component_map[hostname] url = build_url(host, port, ["download"]) logger.info(f"making a request to url - {url}") response = requests.get(url) - logger.info(f"Response from {service} has {len(response.content)} bytes") - with open(f"{service}.feather", "wb") as out_file: + logger.info(f"Response from {hostname} has {len(response.content)} bytes") + with open(f"{hostname}.feather", "wb") as out_file: out_file.write(response.content) file_path = "results.zip" @@ -156,7 +164,6 @@ def download_results(): except Exception as e: raise HTTPException(status_code=404, detail="Failed download") - @app.get("/terminate") def terminate_simulation(): try: @@ -165,10 +172,14 @@ def terminate_simulation(): except Exception as e: raise HTTPException(status_code=404, detail="Failed download ") +def _get_feeder_info(component_map:dict): + for host in component_map: + if host == "feeder": + return host, component_map[host] def run_simulation(): - services, component_map, broker_ip, api_port = read_settings() - print(services) + component_map, broker_ip, api_port = read_settings() + feeder_host, feeder_port = _get_feeder_info(component_map) logger.info(f"{broker_ip}, {api_port}") initstring = f"-f {len(component_map)} --name=mainbroker --loglevel=trace --local_interface={broker_ip} --localport=23404" logger.info(f"Broker initaialization string: {initstring}") @@ -187,7 +198,8 @@ def run_simulation(): "broker_port": 23404, "broker_ip": broker_ip, "api_port": api_port, - "services": services, + "feeder_host": feeder_host, + "feeder_port": feeder_port } replies.append(grequests.post(url, json=myobj)) grequests.map(replies) @@ -197,7 +209,6 @@ def run_simulation(): return - @app.post("/run") async def run_feeder(background_tasks: BackgroundTasks): try: @@ -207,8 +218,7 @@ async def run_feeder(background_tasks: BackgroundTasks): except Exception as e: err = traceback.format_exc() raise HTTPException(status_code=404, detail=str(err)) - - + @app.post("/configure") async def configure(wiring_diagram:WiringDiagram): json.dump(wiring_diagram.dict(), open(WIRING_DIAGRAM_FILENAME, "w")) @@ -230,3 +240,5 @@ async def configure(wiring_diagram:WiringDiagram): if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) + #test_function() + #read_settings() \ No newline at end of file diff --git a/measuring_federate/server.py b/measuring_federate/server.py index 7ea47c3..f3632d3 100644 --- a/measuring_federate/server.py +++ b/measuring_federate/server.py @@ -40,8 +40,8 @@ async def read_root(): @app.post("/run") async def run_model(broker_config:BrokerConfig, background_tasks: BackgroundTasks): logging.info(broker_config) - feeder_host = broker_config.services['oedisi_feeder']['hostname'] - feeder_port = int(broker_config.services['oedisi_feeder']['ports'][0].split(":")[0]) + feeder_host = broker_config.feeder_host + feeder_port = broker_config.feeder_port url = build_url(feeder_host, feeder_port, ['sensor']) logging.info(url) try: From 35d2e195d05e3ef4922ff51f3f964f56a01412e5 Mon Sep 17 00:00:00 2001 From: Latif Date: Fri, 29 Mar 2024 12:00:12 -0600 Subject: [PATCH 08/13] logger fix --- broker/server.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/broker/server.py b/broker/server.py index 2a8629c..d9d1a2a 100644 --- a/broker/server.py +++ b/broker/server.py @@ -19,10 +19,8 @@ from oedisi.componentframework.system_configuration import WiringDiagram, ComponentStruct from oedisi.types.common import ServerReply, HeathCheck - -#logger = logging.getLogger('uvicorn.error') -logger = logging.getLogger(__name__) - +logger = logging.getLogger('uvicorn.error') +#logger = logging.getLogger(__name__) app = FastAPI() @@ -31,8 +29,6 @@ WIRING_DIAGRAM_FILENAME = "system.json" WIRING_DIAGRAM : WiringDiagram | None = None - - # def test_function(json_file = r"C:/Users/alatif/Documents/GitHub/sgidal-example/scenarios/system.json"): # global WIRING_DIAGRAM # data = json.load(open(json_file, "r")) @@ -62,8 +58,6 @@ def read_settings(): broker_host: api_port } if WIRING_DIAGRAM: - print(WIRING_DIAGRAM) - print("") for component in WIRING_DIAGRAM.components: component_map[component.host] = component.container_port else: @@ -221,6 +215,9 @@ async def run_feeder(background_tasks: BackgroundTasks): @app.post("/configure") async def configure(wiring_diagram:WiringDiagram): + global WIRING_DIAGRAM + WIRING_DIAGRAM = wiring_diagram + json.dump(wiring_diagram.dict(), open(WIRING_DIAGRAM_FILENAME, "w")) for component in wiring_diagram.components: component_model = ComponentStruct( @@ -240,5 +237,5 @@ async def configure(wiring_diagram:WiringDiagram): if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=int(os.environ['PORT'])) - #test_function() - #read_settings() \ No newline at end of file + # test_function() + # read_settings() \ No newline at end of file From de5a1a40cb9d8046761eb88d23a97bde5d2129d3 Mon Sep 17 00:00:00 2001 From: Latif Date: Fri, 29 Mar 2024 12:11:11 -0600 Subject: [PATCH 09/13] bug fix for broker serveer --- broker/server.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/broker/server.py b/broker/server.py index d9d1a2a..e040917 100644 --- a/broker/server.py +++ b/broker/server.py @@ -175,7 +175,7 @@ def run_simulation(): component_map, broker_ip, api_port = read_settings() feeder_host, feeder_port = _get_feeder_info(component_map) logger.info(f"{broker_ip}, {api_port}") - initstring = f"-f {len(component_map)} --name=mainbroker --loglevel=trace --local_interface={broker_ip} --localport=23404" + initstring = f"-f {len(component_map)-1} --name=mainbroker --loglevel=trace --local_interface={broker_ip} --localport=23404" logger.info(f"Broker initaialization string: {initstring}") broker = h.helicsCreateBroker("zmq", "", initstring) logger.info(broker) @@ -183,19 +183,22 @@ def run_simulation(): logger.info(f"Broker connected: {isconnected}") logger.info(str(component_map)) replies = [] + + broker_host = socket.gethostname() + for service_ip, service_port in component_map.items(): - - url = build_url(service_ip, service_port, ["run"]) - logger.info(f"making a request to url - {url}") - - myobj = { - "broker_port": 23404, - "broker_ip": broker_ip, - "api_port": api_port, - "feeder_host": feeder_host, - "feeder_port": feeder_port - } - replies.append(grequests.post(url, json=myobj)) + if service_ip != broker_host: + url = build_url(service_ip, service_port, ["run"]) + logger.info(f"making a request to url - {url}") + + myobj = { + "broker_port": 23404, + "broker_ip": broker_ip, + "api_port": api_port, + "feeder_host": feeder_host, + "feeder_port": feeder_port + } + replies.append(grequests.post(url, json=myobj)) grequests.map(replies) while h.helicsBrokerIsConnected(broker): time.sleep(1) From deb5bd49045f035f40adcfb105b0d32f6c2b6a47 Mon Sep 17 00:00:00 2001 From: Latif Date: Thu, 4 Apr 2024 10:05:40 -0600 Subject: [PATCH 10/13] fixed hostname parenthesis issue --- broker/server.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/broker/server.py b/broker/server.py index e040917..e7fae10 100644 --- a/broker/server.py +++ b/broker/server.py @@ -29,11 +29,6 @@ WIRING_DIAGRAM_FILENAME = "system.json" WIRING_DIAGRAM : WiringDiagram | None = None -# def test_function(json_file = r"C:/Users/alatif/Documents/GitHub/sgidal-example/scenarios/system.json"): -# global WIRING_DIAGRAM -# data = json.load(open(json_file, "r")) -# WIRING_DIAGRAM = WiringDiagram(**data) - def build_url(host:str, port:int, enpoint:list): if is_kubernetes_env: KUBERNETES_SERVICE_NAME = os.environ['KUBERNETES_SERVICE_NAME'] @@ -80,7 +75,7 @@ async def upload_profiles(file: UploadFile): try: component_map, _, _ = read_settings() for hostname in component_map: - if "feeder" in hostname(): + if "feeder" in hostname: ip = hostname port = component_map[hostname] data = file.file.read() @@ -108,7 +103,7 @@ async def upload_model(file: UploadFile): try: component_map, _, _ = read_settings() for hostname in component_map: - if "feeder" in hostname(): + if "feeder" in hostname: ip = hostname port = component_map[hostname] data = file.file.read() @@ -136,7 +131,7 @@ def download_results(): component_map, _, _ = read_settings() for hostname in component_map: - if "recorder" in hostname(): + if "recorder" in hostname: host = hostname port = component_map[hostname] From 92d17aed4b641066c516d59bbccefe7ddfed4dc7 Mon Sep 17 00:00:00 2001 From: Latif Date: Tue, 16 Apr 2024 12:25:28 -0600 Subject: [PATCH 11/13] updating requirements.txt files --- LocalFeeder/requirements.txt | 2 +- broker/requirements.txt | 2 +- measuring_federate/requirements.txt | 2 +- recorder/requirements.txt | 2 +- wls_federate/requirements.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/LocalFeeder/requirements.txt b/LocalFeeder/requirements.txt index 1256674..2a244ea 100644 --- a/LocalFeeder/requirements.txt +++ b/LocalFeeder/requirements.txt @@ -11,5 +11,5 @@ boto3 xarray fastapi uvicorn -git+https://github.com/openEDI/oedisi@al/config +oedisi python-multipart \ No newline at end of file diff --git a/broker/requirements.txt b/broker/requirements.txt index 8b07bec..ef7ac5e 100644 --- a/broker/requirements.txt +++ b/broker/requirements.txt @@ -3,6 +3,6 @@ helics-apps==3.4.0 pyyaml fastapi uvicorn -git+https://github.com/openEDI/oedisi@al/config +oedisi grequests python-multipart diff --git a/measuring_federate/requirements.txt b/measuring_federate/requirements.txt index 1348db6..6a0e3f2 100644 --- a/measuring_federate/requirements.txt +++ b/measuring_federate/requirements.txt @@ -8,4 +8,4 @@ fastapi uvicorn requests grequests -git+https://github.com/openEDI/oedisi@al/config +oedisi \ No newline at end of file diff --git a/recorder/requirements.txt b/recorder/requirements.txt index aa469b6..019b05f 100644 --- a/recorder/requirements.txt +++ b/recorder/requirements.txt @@ -6,4 +6,4 @@ numpy pandas fastapi uvicorn -git+https://github.com/openEDI/oedisi@al/config +oedisi \ No newline at end of file diff --git a/wls_federate/requirements.txt b/wls_federate/requirements.txt index cae4e56..89dd33d 100644 --- a/wls_federate/requirements.txt +++ b/wls_federate/requirements.txt @@ -5,4 +5,4 @@ scipy numpy fastapi uvicorn -git+https://github.com/openEDI/oedisi@al/config +oedisi \ No newline at end of file From 3ac06b6ae0ebdb71d0f65a2541c410ceb88ac38a Mon Sep 17 00:00:00 2001 From: Latif Date: Tue, 16 Apr 2024 13:11:10 -0600 Subject: [PATCH 12/13] updating docker files --- LocalFeeder/Dockerfile | 10 ++++++++++ broker/Dockerfile | 2 +- measuring_federate/Dockerfile | 10 ++++++++++ recorder/Dockerfile | 10 ++++++++++ wls_federate/Dockerfile | 18 ++++++------------ 5 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 LocalFeeder/Dockerfile create mode 100644 measuring_federate/Dockerfile create mode 100644 recorder/Dockerfile diff --git a/LocalFeeder/Dockerfile b/LocalFeeder/Dockerfile new file mode 100644 index 0000000..92163e3 --- /dev/null +++ b/LocalFeeder/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10.6-slim-bullseye +RUN apt-get update +RUN apt-get install -y git ssh +RUN mkdir LocalFeeder +COPY . ./LocalFeeder +WORKDIR ./LocalFeeder +RUN pip cache purge +RUN pip install -r requirements.txt +EXPOSE 5678/tcp +CMD ["python", "server.py"] diff --git a/broker/Dockerfile b/broker/Dockerfile index 0c009d8..a10fe17 100644 --- a/broker/Dockerfile +++ b/broker/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get install -y git ssh RUN mkdir broker COPY * ./broker WORKDIR ./broker - +RUN pip cache purge RUN pip install -r requirements.txt EXPOSE 8766/tcp diff --git a/measuring_federate/Dockerfile b/measuring_federate/Dockerfile new file mode 100644 index 0000000..c3e52d4 --- /dev/null +++ b/measuring_federate/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10.6-slim-bullseye +RUN apt-get update +RUN apt-get install -y git ssh +RUN mkdir MeasurementComponent +COPY . ./MeasurementComponent +WORKDIR ./MeasurementComponent +RUN pip cache purge +RUN pip install -r requirements.txt +EXPOSE 5684/tcp +CMD ["python", "server.py"] diff --git a/recorder/Dockerfile b/recorder/Dockerfile new file mode 100644 index 0000000..3f81bb0 --- /dev/null +++ b/recorder/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10.6-slim-bullseye +RUN apt-get update +RUN apt-get install -y git ssh +RUN mkdir Recorder +COPY . ./Recorder +WORKDIR ./Recorder +RUN pip cache purge +RUN pip install -r requirements.txt +EXPOSE 5679/tcp +CMD ["python", "server.py"] diff --git a/wls_federate/Dockerfile b/wls_federate/Dockerfile index ebceb7f..0b052d8 100644 --- a/wls_federate/Dockerfile +++ b/wls_federate/Dockerfile @@ -1,16 +1,10 @@ FROM python:3.10.6-slim-bullseye - -RUN apt-get update +RUN apt-get update RUN apt-get install -y git ssh - - -RUN mkdir recorder_federate -COPY * ./recorder_federate -WORKDIR ./recorder_federate - +RUN mkdir StateEstimatorComponent +COPY . ./StateEstimatorComponent +WORKDIR ./StateEstimatorComponent +RUN pip cache purge RUN pip install -r requirements.txt - -EXPOSE 8766/tcp - +EXPOSE 5683/tcp CMD ["python", "server.py"] - From 790ea5a8ce25b8aec5e8e7370cb59d96132b2300 Mon Sep 17 00:00:00 2001 From: Latif Date: Tue, 16 Apr 2024 13:22:29 -0600 Subject: [PATCH 13/13] removing pip cache --- LocalFeeder/Dockerfile | 1 - broker/Dockerfile | 2 +- measuring_federate/Dockerfile | 1 - recorder/Dockerfile | 1 - wls_federate/Dockerfile | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/LocalFeeder/Dockerfile b/LocalFeeder/Dockerfile index 92163e3..8a2427b 100644 --- a/LocalFeeder/Dockerfile +++ b/LocalFeeder/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get install -y git ssh RUN mkdir LocalFeeder COPY . ./LocalFeeder WORKDIR ./LocalFeeder -RUN pip cache purge RUN pip install -r requirements.txt EXPOSE 5678/tcp CMD ["python", "server.py"] diff --git a/broker/Dockerfile b/broker/Dockerfile index a10fe17..0c009d8 100644 --- a/broker/Dockerfile +++ b/broker/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get install -y git ssh RUN mkdir broker COPY * ./broker WORKDIR ./broker -RUN pip cache purge + RUN pip install -r requirements.txt EXPOSE 8766/tcp diff --git a/measuring_federate/Dockerfile b/measuring_federate/Dockerfile index c3e52d4..29f1b71 100644 --- a/measuring_federate/Dockerfile +++ b/measuring_federate/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get install -y git ssh RUN mkdir MeasurementComponent COPY . ./MeasurementComponent WORKDIR ./MeasurementComponent -RUN pip cache purge RUN pip install -r requirements.txt EXPOSE 5684/tcp CMD ["python", "server.py"] diff --git a/recorder/Dockerfile b/recorder/Dockerfile index 3f81bb0..82988ba 100644 --- a/recorder/Dockerfile +++ b/recorder/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get install -y git ssh RUN mkdir Recorder COPY . ./Recorder WORKDIR ./Recorder -RUN pip cache purge RUN pip install -r requirements.txt EXPOSE 5679/tcp CMD ["python", "server.py"] diff --git a/wls_federate/Dockerfile b/wls_federate/Dockerfile index 0b052d8..cc55cb6 100644 --- a/wls_federate/Dockerfile +++ b/wls_federate/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get install -y git ssh RUN mkdir StateEstimatorComponent COPY . ./StateEstimatorComponent WORKDIR ./StateEstimatorComponent -RUN pip cache purge RUN pip install -r requirements.txt EXPOSE 5683/tcp CMD ["python", "server.py"]