From 6bfa0a4a1f5683df65e6ed89d0e95923cff2497c Mon Sep 17 00:00:00 2001 From: Florian Esser Date: Tue, 26 Nov 2024 18:03:33 +0100 Subject: [PATCH] provide comfyUI build persist exact config and commands that lead to offline installer creation Signed-off-by: Florian Esser --- .gitignore | 3 ++- WebUI/build/arc-settings.json | 4 ++-- WebUI/build/build-config-offline.json | 21 ++++++++++++++++ .../scripts/fetch-python-package-resources.js | 24 +++++++++++++++++++ .../build/scripts/install-full-python-env.js | 13 +++++++--- WebUI/build/scripts/prepare-python-env.js | 1 + .../provide-electron-build-resources.js | 17 ++++++++++--- WebUI/package.json | 2 +- service/requirements-arc.txt | 5 ++-- 9 files changed, 77 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 633c2e4b..29870908 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ .vscode/ env/ package_res/ -python_package_res/ +build_resources/ +build_envs/ release/ WebUI/external/service/ diff --git a/WebUI/build/arc-settings.json b/WebUI/build/arc-settings.json index dc319c74..58077c22 100644 --- a/WebUI/build/arc-settings.json +++ b/WebUI/build/arc-settings.json @@ -1,5 +1,5 @@ { "envType": "arc", - "availableThemes": ["dark"], - "currentTheme": "dark" + "availableThemes": ["dark","lnl","bmg"], + "currentTheme": "bmg" } \ No newline at end of file diff --git a/WebUI/build/build-config-offline.json b/WebUI/build/build-config-offline.json index dda33d15..b49d5649 100644 --- a/WebUI/build/build-config-offline.json +++ b/WebUI/build/build-config-offline.json @@ -21,6 +21,27 @@ "*.bat" ] }, + { + "from": "external/ComfyUI", + "to": "ComfyUI", + "filter": [ + "**/*", + "!models/", + "!.vscode/", + "!__pycache__/", + "!test/", + "!db/", + "!cache/", + "!static/", + "!temp/", + "!dist/", + "*.bat" + ] + }, + { + "from": "external/workflows", + "to": "workflows" + }, { "from": "external/service/models/stable_diffusion/ESRGAN", "to": "service/models/stable_diffusion/ESRGAN" diff --git a/WebUI/build/scripts/fetch-python-package-resources.js b/WebUI/build/scripts/fetch-python-package-resources.js index e7847a41..7c45eca9 100644 --- a/WebUI/build/scripts/fetch-python-package-resources.js +++ b/WebUI/build/scripts/fetch-python-package-resources.js @@ -2,6 +2,7 @@ const https = require('https'); const fs = require('fs'); const path = require('path'); +const childProcess = require('child_process'); const argv = require('minimist')(process.argv.slice(2)); @@ -78,12 +79,35 @@ function copyLibuvDllsIfNotPresent() { } } +function fetchComfyUIIfNotPresent() { + const comfyUICloneDir = path.join(targetDir, 'ComfyUI') + if (fs.existsSync(comfyUICloneDir)) { + console.log(`omitting fetching of comfyUI as ${comfyUICloneDir} already exists`) + } else { + gitClone("https://github.com/comfyanonymous/ComfyUI.git", comfyUICloneDir) + gitClone("https://github.com/city96/ComfyUI-GGUF.git", path.join(comfyUICloneDir, 'custom_nodes', 'ComfyUI-GGUF')) + } +} + + +function gitClone(repoURL, targetDir) { + const gitClone = childProcess.spawnSync("git", ["clone", repoURL, targetDir]); + console.log(gitClone.stdout.toString()); + console.error(gitClone.stderr.toString()); + if (gitClone.status!== 0) { + console.error('Failed to clone repo: ', repoURL); + process.exit(1); + } + console.log('Successfully fetched: ', repoURL); +} + function main() { prepareTargetPath() fetchFileIfNotPresent(embeddablePythonUrl) fetchFileIfNotPresent(getPipScriptUrl) fetchFileIfNotPresent(sevenZrExeUrl) + fetchComfyUIIfNotPresent() copyLibuvDllsIfNotPresent() } diff --git a/WebUI/build/scripts/install-full-python-env.js b/WebUI/build/scripts/install-full-python-env.js index 8a97028a..79154d85 100644 --- a/WebUI/build/scripts/install-full-python-env.js +++ b/WebUI/build/scripts/install-full-python-env.js @@ -8,13 +8,15 @@ const childProcess = require('child_process'); const argv = require('minimist')(process.argv.slice(2)); const envDirArg = argv.env_dir const platformArg = argv.platform +const comfyUIDIrArg = argv.comfy_ui_dir -if (!envDirArg || !platformArg) { +if (!envDirArg || !platformArg || !comfyUIDIrArg) { console.error('Usage: node install-full-python-env.js --env_dir=$DIR ---platform=arc|ultra|ultra2\n'); process.exit(1); } const envDir = existingFileOrExit(path.resolve(envDirArg)); +const comfyUIDIr = existingFileOrExit(path.resolve(comfyUIDIrArg)); const platform = platformArg; function existingFileOrExit(filePath) { @@ -29,7 +31,7 @@ function existingFileOrExit(filePath) { function installPip(pythonExe, getPipFilePath) { const runGetPip = childProcess.spawnSync(pythonExe, [getPipFilePath]); console.log(runGetPip.stdout.toString()); - //console.error(runGetPip.stderr.toString()); + console.error(runGetPip.stderr.toString()); if (runGetPip.status!== 0) { console.error('Failed to install requirements'); process.exit(1); @@ -56,7 +58,7 @@ function copyToTargetDir(sourceDir, targetDir) { function prepareTargetDir(targetDir) { if (fs.existsSync(targetDir)) { - console.log("clearing previous env dir ${targetDir}") + console.log(`clearing previous env dir ${targetDir}`) fs.rmSync(targetDir, { recursive: true }); } } @@ -72,10 +74,15 @@ function main() { const platformSpecificRequirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..','..', 'service', `requirements-${platform}.txt`)); const requirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', '..', 'service', `requirements.txt`)); + const comfyUiRequirementsTxt = existingFileOrExit(path.join(comfyUIDIr, `requirements.txt`)); + const ggufCostomNoderequirementsTxt = existingFileOrExit(path.join(comfyUIDIr, 'custom_nodes', 'ComfyUI-GGUF', `requirements.txt`)); + installPip(pythonExe, getPipFile) runPipInstall(pythonExe, platformSpecificRequirementsTxt) runPipInstall(pythonExe, requirementsTxt) + runPipInstall(pythonExe, comfyUiRequirementsTxt) + runPipInstall(pythonExe, ggufCostomNoderequirementsTxt) } main(); diff --git a/WebUI/build/scripts/prepare-python-env.js b/WebUI/build/scripts/prepare-python-env.js index 517e5509..1f14f8f4 100644 --- a/WebUI/build/scripts/prepare-python-env.js +++ b/WebUI/build/scripts/prepare-python-env.js @@ -71,6 +71,7 @@ function createPythonEnvFromEmbedabblePythonZip(targetDir, pythonEmbedZipFile) { python311.zip . ../service +../ComfyUI # Uncomment to run site.main() automatically import site diff --git a/WebUI/build/scripts/provide-electron-build-resources.js b/WebUI/build/scripts/provide-electron-build-resources.js index 268ef2bb..4e46f74b 100644 --- a/WebUI/build/scripts/provide-electron-build-resources.js +++ b/WebUI/build/scripts/provide-electron-build-resources.js @@ -48,14 +48,20 @@ function zipPythonEnv(sevenZipExe, pythonEnvDir, targetPath) { console.log('Offline env has been compressed to:', targetPath); } -function copyResources(targetDir, ...files) { +function copyFiles(targetDir, ...files) { for (const file of files) { - console.log(file) fs.copyFileSync(file, path.join(targetDir, path.basename(file))); console.log('Copied:', file, 'to:', path.join(targetDir, path.basename(file))); } } +function copyDirectories(targetDir, ...dirs) { + for (const dir of dirs) { + fs.cpSync(dir, path.join(targetDir, path.basename(dir)), { recursive: true }); + console.log('Copied:', dir, 'to:', path.join(targetDir, path.basename(dir))); + } +} + function clearPreviousZip(zipFilePath) { if (fs.existsSync(zipFilePath)) { @@ -66,14 +72,19 @@ function clearPreviousZip(zipFilePath) { function main() { const sevenZipExe = path.join(buildResourcesDir, '7zr.exe'); + const comfyUI = path.join(buildResourcesDir, 'ComfyUI'); clearPreviousZip(path.join(targetDir, `env.7z`)); zipPythonEnv(sevenZipExe, pythenEnvDir, path.join(targetDir, `env.7z`)); symlinkBackendDir(backendDir, path.join(targetDir, 'service')) - copyResources(targetDir, + copyFiles(targetDir, sevenZipExe ) + + copyDirectories(targetDir, + comfyUI + ) } main() diff --git a/WebUI/package.json b/WebUI/package.json index 5c8477ab..cac1a0ac 100644 --- a/WebUI/package.json +++ b/WebUI/package.json @@ -9,7 +9,7 @@ "fetch-build-resources": "cross-env node ./build/scripts/fetch-python-package-resources.js --target_dir=../build_resources", "prepare-python-env": "cross-env node ./build/scripts/prepare-python-env.js --build_resources_dir=../build_resources --target_dir=../build-envs/online/env", - "install-full-python-env": "cross-env node ./build/scripts/install-full-python-env.js --env_dir=../build-envs/online/env", + "install-full-python-env": "cross-env node ./build/scripts/install-full-python-env.js --env_dir=../build-envs/online/env --comfy_ui_dir=../build_resources/ComfyUI", "provide-electron-build-resources": "cross-env node build/scripts/provide-electron-build-resources.js --build_resources_dir=../build_resources --backend_dir=../service --target_dir=./external", "prepare-build:arc": "cross-env npm run prepare-python-env && npm run provide-electron-build-resources -- --python_env_dir=../build-envs/online/env && node ./build/scripts/render-template.js --online_installer --platform=arc", diff --git a/service/requirements-arc.txt b/service/requirements-arc.txt index 6b933d39..a56071bc 100644 --- a/service/requirements-arc.txt +++ b/service/requirements-arc.txt @@ -5,8 +5,7 @@ mkl-dpcpp==2024.2.1 torch==2.3.1.post0+cxx11.abi torchvision==0.18.1.post0+cxx11.abi torchaudio==2.3.1.post0+cxx11.abi -intel-extension-for-pytorch==2.3.110.post0+xpu -mkl-dpcpp==2024.2.1 +C:\intel_build_tests\ipex23_wheels\intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl # IPEX-LLM --pre @@ -14,4 +13,4 @@ ipex_llm==2.2.0b2 bigdl-core-xe-23==2.6.0b2 bigdl-core-xe-addons-23==2.6.0b2 bigdl-core-xe-batch-23==2.6.0b2 -onednn-devel==2024.1.1 +onednn-devel==2024.1.1 \ No newline at end of file