provide comfyUI build

persist exact config and commands that lead to offline installer creation

Signed-off-by: Florian Esser <[email protected]>

.gitignore

@@ -1,7 +1,8 @@
 .vscode/
 env/
 package_res/
-python_package_res/
+build_resources/
+build_envs/
 release/
 WebUI/external/service/
 

WebUI/build/arc-settings.json

@@ -1,5 +1,5 @@
 {
     "envType": "arc",
-    "availableThemes": ["dark"],
-    "currentTheme": "dark"
+    "availableThemes": ["dark","lnl","bmg"],
+    "currentTheme": "bmg"
 }

WebUI/build/build-config-offline.json

@@ -21,6 +21,27 @@
         "*.bat"
       ]
     },
+    {
+      "from": "external/ComfyUI",
+      "to": "ComfyUI",
+      "filter": [
+        "**/*",
+        "!models/",
+        "!.vscode/",
+        "!__pycache__/",
+        "!test/",
+        "!db/",
+        "!cache/",
+        "!static/",
+        "!temp/",
+        "!dist/",
+        "*.bat"
+      ]
+    },
+    {
+      "from": "external/workflows",
+      "to": "workflows"
+    },
     {
       "from": "external/service/models/stable_diffusion/ESRGAN",
       "to": "service/models/stable_diffusion/ESRGAN"

WebUI/build/scripts/fetch-python-package-resources.js

@@ -2,6 +2,7 @@
 const https = require('https');
 const fs = require('fs');
 const path = require('path');
+const childProcess = require('child_process');
 
 
 const argv = require('minimist')(process.argv.slice(2));
@@ -78,12 +79,35 @@ function copyLibuvDllsIfNotPresent() {
     }
 }
 
+function fetchComfyUIIfNotPresent() {
+    const comfyUICloneDir = path.join(targetDir, 'ComfyUI')
+    if (fs.existsSync(comfyUICloneDir)) {
+        console.log(`omitting fetching of comfyUI as ${comfyUICloneDir} already exists`)
+    } else {
+        gitClone("https://github.com/comfyanonymous/ComfyUI.git", comfyUICloneDir)
+        gitClone("https://github.com/city96/ComfyUI-GGUF.git", path.join(comfyUICloneDir, 'custom_nodes', 'ComfyUI-GGUF'))
+    }
+}
+
+
+function gitClone(repoURL, targetDir) {
+    const gitClone = childProcess.spawnSync("git", ["clone", repoURL, targetDir]);
+    console.log(gitClone.stdout.toString());
+    console.error(gitClone.stderr.toString());
+    if (gitClone.status!== 0) {
+        console.error('Failed to clone repo: ', repoURL);
+        process.exit(1);
+    }
+    console.log('Successfully fetched: ', repoURL);
+}
+
 
 function main() {
     prepareTargetPath()
     fetchFileIfNotPresent(embeddablePythonUrl)
     fetchFileIfNotPresent(getPipScriptUrl)
     fetchFileIfNotPresent(sevenZrExeUrl)
+    fetchComfyUIIfNotPresent()
     copyLibuvDllsIfNotPresent()
 }
 

WebUI/build/scripts/install-full-python-env.js

@@ -8,13 +8,15 @@ const childProcess = require('child_process');
 const argv = require('minimist')(process.argv.slice(2));
 const envDirArg = argv.env_dir
 const platformArg = argv.platform
+const comfyUIDIrArg = argv.comfy_ui_dir
 
-if (!envDirArg || !platformArg) {
+if (!envDirArg || !platformArg || !comfyUIDIrArg) {
     console.error('Usage: node install-full-python-env.js --env_dir=$DIR ---platform=arc|ultra|ultra2\n');
     process.exit(1);
 }
 
 const envDir = existingFileOrExit(path.resolve(envDirArg));
+const comfyUIDIr = existingFileOrExit(path.resolve(comfyUIDIrArg));
 const platform = platformArg;
 
 function existingFileOrExit(filePath) {
@@ -29,7 +31,7 @@ function existingFileOrExit(filePath) {
 function installPip(pythonExe, getPipFilePath) {
     const runGetPip = childProcess.spawnSync(pythonExe, [getPipFilePath]);
     console.log(runGetPip.stdout.toString());
-    //console.error(runGetPip.stderr.toString());
+    console.error(runGetPip.stderr.toString());
     if (runGetPip.status!== 0) {
         console.error('Failed to install requirements');
         process.exit(1);
@@ -56,7 +58,7 @@ function copyToTargetDir(sourceDir, targetDir) {
 
 function prepareTargetDir(targetDir) {
     if (fs.existsSync(targetDir)) {
-        console.log("clearing previous env dir ${targetDir}")
+        console.log(`clearing previous env dir ${targetDir}`)
         fs.rmSync(targetDir, { recursive: true });
     }
 }
@@ -72,10 +74,15 @@ function main() {
 
     const platformSpecificRequirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..','..', 'service', `requirements-${platform}.txt`));
     const requirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', '..', 'service', `requirements.txt`));
+    const comfyUiRequirementsTxt = existingFileOrExit(path.join(comfyUIDIr, `requirements.txt`));
+    const ggufCostomNoderequirementsTxt = existingFileOrExit(path.join(comfyUIDIr, 'custom_nodes', 'ComfyUI-GGUF', `requirements.txt`));
+
 
     installPip(pythonExe, getPipFile)
     runPipInstall(pythonExe, platformSpecificRequirementsTxt)
     runPipInstall(pythonExe, requirementsTxt)
+    runPipInstall(pythonExe, comfyUiRequirementsTxt)
+    runPipInstall(pythonExe, ggufCostomNoderequirementsTxt)
 }
 
 main();

WebUI/build/scripts/prepare-python-env.js

@@ -71,6 +71,7 @@ function createPythonEnvFromEmbedabblePythonZip(targetDir, pythonEmbedZipFile) {
 python311.zip
 .
 ../service
+../ComfyUI
 
 # Uncomment to run site.main() automatically
 import site

WebUI/build/scripts/provide-electron-build-resources.js

@@ -48,14 +48,20 @@ function zipPythonEnv(sevenZipExe, pythonEnvDir, targetPath) {
     console.log('Offline env has been compressed to:', targetPath);
 }
 
-function copyResources(targetDir, ...files) {
+function copyFiles(targetDir, ...files) {
     for (const file of files) {
-        console.log(file)
         fs.copyFileSync(file, path.join(targetDir, path.basename(file)));
         console.log('Copied:', file, 'to:', path.join(targetDir, path.basename(file)));
     }
 }
 
+function copyDirectories(targetDir, ...dirs) {
+    for (const dir of dirs) {
+        fs.cpSync(dir, path.join(targetDir, path.basename(dir)), { recursive: true });
+        console.log('Copied:', dir, 'to:', path.join(targetDir, path.basename(dir)));
+    }
+}
+
 function clearPreviousZip(zipFilePath) {
     if
     (fs.existsSync(zipFilePath)) {
@@ -66,14 +72,19 @@ function clearPreviousZip(zipFilePath) {
 
 function main() {
     const sevenZipExe = path.join(buildResourcesDir, '7zr.exe');
+    const comfyUI = path.join(buildResourcesDir, 'ComfyUI');
 
     clearPreviousZip(path.join(targetDir, `env.7z`));
     zipPythonEnv(sevenZipExe, pythenEnvDir, path.join(targetDir, `env.7z`));
 
     symlinkBackendDir(backendDir, path.join(targetDir, 'service'))
-    copyResources(targetDir,
+    copyFiles(targetDir,
         sevenZipExe
     )
+
+    copyDirectories(targetDir,
+        comfyUI
+    )
 }
 
 main()

WebUI/package.json

@@ -9,7 +9,7 @@
     "fetch-build-resources": "cross-env node ./build/scripts/fetch-python-package-resources.js --target_dir=../build_resources",
 
     "prepare-python-env": "cross-env node ./build/scripts/prepare-python-env.js --build_resources_dir=../build_resources --target_dir=../build-envs/online/env",
-    "install-full-python-env": "cross-env node ./build/scripts/install-full-python-env.js --env_dir=../build-envs/online/env",
+    "install-full-python-env": "cross-env node ./build/scripts/install-full-python-env.js --env_dir=../build-envs/online/env --comfy_ui_dir=../build_resources/ComfyUI",
     "provide-electron-build-resources": "cross-env node build/scripts/provide-electron-build-resources.js --build_resources_dir=../build_resources --backend_dir=../service --target_dir=./external",
 
     "prepare-build:arc": "cross-env npm run prepare-python-env && npm run provide-electron-build-resources -- --python_env_dir=../build-envs/online/env && node ./build/scripts/render-template.js --online_installer --platform=arc",

service/requirements-arc.txt

@@ -5,13 +5,12 @@ mkl-dpcpp==2024.2.1
 torch==2.3.1.post0+cxx11.abi
 torchvision==0.18.1.post0+cxx11.abi
 torchaudio==2.3.1.post0+cxx11.abi
-intel-extension-for-pytorch==2.3.110.post0+xpu
-mkl-dpcpp==2024.2.1
+C:\intel_build_tests\ipex23_wheels\intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl
 
 # IPEX-LLM
 --pre
 ipex_llm==2.2.0b2
 bigdl-core-xe-23==2.6.0b2
 bigdl-core-xe-addons-23==2.6.0b2
 bigdl-core-xe-batch-23==2.6.0b2
-onednn-devel==2024.1.1
+onednn-devel==2024.1.1