From 44490bd7daabad139dd0944e3486ed06e9fc03ac Mon Sep 17 00:00:00 2001 From: Florian Esser Date: Tue, 26 Nov 2024 14:36:49 +0100 Subject: [PATCH] Refactor build scripts provide scripts, that simply setup a python env This env may be used for developers (on windows machines) Overall flow remains equal, but try to make things a little more obvious my adding named parameters of scripts. --- WebUI/build/after_build.bat | 11 -- WebUI/build/arc_before_build.bat | 19 --- WebUI/build/before_build.bat | 7 -- ...ine.nsh => installer-offline.nsh.template} | 6 +- WebUI/build/installer.nsh | 59 ---------- WebUI/build/installer.nsh.template | 22 +++- .../scripts/fetch-python-package-resources.js | 51 +++++--- .../build/scripts/install-full-python-env.js | 73 ++++++++++++ WebUI/build/scripts/pack-offline.js | 111 ------------------ WebUI/build/scripts/prebuild.js | 63 ---------- .../{pack-python.js => prepare-python-env.js} | 40 ++++--- .../provide-electron-build-resources.js | 69 +++++++++++ WebUI/build/scripts/render-template.js | 65 ++++++---- WebUI/build/ultra_before_build.bat | 19 --- WebUI/package.json | 23 ++-- service/requirements-arc.txt | 3 +- 16 files changed, 282 insertions(+), 359 deletions(-) delete mode 100644 WebUI/build/after_build.bat delete mode 100644 WebUI/build/arc_before_build.bat delete mode 100644 WebUI/build/before_build.bat rename WebUI/build/{installer-offline.nsh => installer-offline.nsh.template} (86%) delete mode 100644 WebUI/build/installer.nsh create mode 100644 WebUI/build/scripts/install-full-python-env.js delete mode 100644 WebUI/build/scripts/pack-offline.js delete mode 100644 WebUI/build/scripts/prebuild.js rename WebUI/build/scripts/{pack-python.js => prepare-python-env.js} (74%) create mode 100644 WebUI/build/scripts/provide-electron-build-resources.js delete mode 100644 WebUI/build/ultra_before_build.bat diff --git a/WebUI/build/after_build.bat b/WebUI/build/after_build.bat deleted file mode 100644 index 898b3c8c..00000000 --- a/WebUI/build/after_build.bat +++ /dev/null @@ -1,11 +0,0 @@ -@echo off -IF EXIST ".\external\service" ( - rd /q /s ".\external\service" -) -IF EXIST ".\external\env.7z" ( - del /F /Q ".\external\env.7z" -) - -IF EXIST ".\external\env" ( - rd /q /s ".\external\env" -) \ No newline at end of file diff --git a/WebUI/build/arc_before_build.bat b/WebUI/build/arc_before_build.bat deleted file mode 100644 index 4f4ac18b..00000000 --- a/WebUI/build/arc_before_build.bat +++ /dev/null @@ -1,19 +0,0 @@ -@echo off -if EXIST ".\external\service" ( - rd /s /q ".\external\service" -) -mklink /J ".\external\service" "..\service" - -if NOT EXIST "..\package_res\arc_env.7z" ( - setlocal enabledelayedexpansion - pushd "..\package_res" - set "absolutePath=!cd!" - echo "You must place arc_env.7z under the directory !absolutePath!" - popd - endlocal - Exit -1 -) -IF EXIST ".\external\env.7z" ( - del /F /Q ".\external\env.7z" -) -mklink /H ".\external\env.7z" "..\package_res\arc_env.7z" \ No newline at end of file diff --git a/WebUI/build/before_build.bat b/WebUI/build/before_build.bat deleted file mode 100644 index 990127e4..00000000 --- a/WebUI/build/before_build.bat +++ /dev/null @@ -1,7 +0,0 @@ -@echo off -if NOT EXIST "./external/service" ( - mklink /J "./external/service" "../service" -) -IF NOT EXIST "../external/env" ( - mklink /J "../external/env" "../env" -) \ No newline at end of file diff --git a/WebUI/build/installer-offline.nsh b/WebUI/build/installer-offline.nsh.template similarity index 86% rename from WebUI/build/installer-offline.nsh rename to WebUI/build/installer-offline.nsh.template index b0e253b5..9c1f49c0 100644 --- a/WebUI/build/installer-offline.nsh +++ b/WebUI/build/installer-offline.nsh.template @@ -2,13 +2,13 @@ ShowInstDetails show !macroend !macro customInstall - IfFileExists "$INSTDIR\resources\env-offline.7z" extracting end + IfFileExists "$INSTDIR\resources\env.7z" extracting end extracting: SetDetailsPrint textonly DetailPrint "Extracting python environment..." - nsExec::ExecToLog '"$INSTDIR\resources\7zr.exe" x "$INSTDIR\resources\env-offline.7z" -o"$INSTDIR\resources"' - Delete "$INSTDIR\resources\env-offline.7z" + nsExec::ExecToLog '"$INSTDIR\resources\7zr.exe" x "$INSTDIR\resources\env.7z" -o"$INSTDIR\resources"' + Delete "$INSTDIR\resources\env.7z" Delete "$INSTDIR\resources\7zr.exe" end: diff --git a/WebUI/build/installer.nsh b/WebUI/build/installer.nsh deleted file mode 100644 index a1949fac..00000000 --- a/WebUI/build/installer.nsh +++ /dev/null @@ -1,59 +0,0 @@ -!macro customHeader - ShowInstDetails show -!macroend -!macro customInstall - IfFileExists "$INSTDIR\resources\env.7z" extracting end - - extracting: - SetDetailsPrint textonly - DetailPrint "Extracting python environment..." - nsExec::ExecToLog '"$INSTDIR\resources\7zr.exe" x "$INSTDIR\resources\env.7z" -o"$INSTDIR\resources"' - Delete "$INSTDIR\resources\env.7z" - Delete "$INSTDIR\resources\7zr.exe" - - DetailPrint "Downloading and setting up python environment..." - ; Display a popup message with OK/Cancel options - MessageBox MB_ICONQUESTION|MB_OKCANCEL "Before installing Python dependencies, please ensure you have a stable internet connection. This could take several minutes. Click OK to proceed or Cancel to abort installation." - - ; Check the return value from MessageBox - StrCmp $R0 IDCANCEL cancel - Goto continue - - cancel: - Abort ; Abort the installation if Cancel is clicked - - continue: - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "-r" "$INSTDIR\resources\service\requirements-arc.txt"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\bigdl_core_xe_23-2.6.0.dev0-cp311-cp311-win_amd64.whl"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\bigdl_core_xe_addons_23-2.6.0.dev0-cp311-cp311-win_amd64.whl"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\bigdl_core_xe_batch_23-2.6.0.dev0-cp311-cp311-win_amd64.whl"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\torch-2.3.1+cxx11.abi-cp311-cp311-win_amd64.whl"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\torchaudio-2.3.1+cxx11.abi-cp311-cp311-win_amd64.whl"' - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\torchvision-0.18.1+cxx11.abi-cp311-cp311-win_amd64.whl"' - end: - SetDetailsPrint both -!macroend - - - -!macro customRemoveFiles - ; Ask the user if they want to keep the models - MessageBox MB_YESNO "Do you want to keep the models directory?" IDYES keepModels IDNO deleteAll - - deleteAll: - ; If the user clicked "Yes", delete the entire installation directory - RMDir /r "$INSTDIR" - Goto end - - keepModels: - ; If the user clicked "No", move the models directory to a temporary location in the same drive, delete the installation directory, and then move back the models directory - StrCpy $0 "$INSTDIR" - StrCpy $1 "_model_backup" - StrCpy $2 "$0$1" - Rename "$INSTDIR\resources\service\models" "$2" - RMDir /r "$INSTDIR" - MessageBox MB_OK "backup model directory at $2" - - end: -!macroend \ No newline at end of file diff --git a/WebUI/build/installer.nsh.template b/WebUI/build/installer.nsh.template index fd889726..d804d13a 100644 --- a/WebUI/build/installer.nsh.template +++ b/WebUI/build/installer.nsh.template @@ -13,7 +13,27 @@ DetailPrint "Downloading and setting up python environment..." ; Display a popup message with OK/Cancel options - + MessageBox MB_ICONQUESTION|MB_OKCANCEL "Before installing Python dependencies, please ensure you have a stable internet connection. This could take several minutes. Click OK to proceed or Cancel to abort installation." + + ; Check the return value from MessageBox + StrCmp $R0 IDCANCEL cancel + Goto continue + + cancel: + Abort ; Abort the installation if Cancel is clicked + + continue: + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "$INSTDIR\resources\env\get-pip.py" "--no-warn-script-location"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\bigdl_core_xe_23-2.6.0.dev0-cp311-cp311-win_amd64.whl"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\bigdl_core_xe_addons_23-2.6.0.dev0-cp311-cp311-win_amd64.whl"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\bigdl_core_xe_batch_23-2.6.0.dev0-cp311-cp311-win_amd64.whl"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\torch-2.3.1+cxx11.abi-cp311-cp311-win_amd64.whl"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\torchaudio-2.3.1+cxx11.abi-cp311-cp311-win_amd64.whl"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "$INSTDIR\resources\arc-ipex-wheels\torchvision-0.18.1+cxx11.abi-cp311-cp311-win_amd64.whl"' + + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "-r" "$INSTDIR\resources\service\requirements-${PLATFORM}.txt" "--no-warn-script-location"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "-r" "$INSTDIR\resources\service\requirements.txt" "--no-warn-script-location"' end: DetailPrint "Installation completed." !macroend diff --git a/WebUI/build/scripts/fetch-python-package-resources.js b/WebUI/build/scripts/fetch-python-package-resources.js index 7b6d6c0b..e7847a41 100644 --- a/WebUI/build/scripts/fetch-python-package-resources.js +++ b/WebUI/build/scripts/fetch-python-package-resources.js @@ -1,31 +1,36 @@ -// Usage: node fetch-python-package-resources.js +// Usage: node fetch-python-package-resources.js --target-dir=$DIR --conda-env-library-dir=$DIR const https = require('https'); const fs = require('fs'); const path = require('path'); -if (process.argv.length < 3) { +const argv = require('minimist')(process.argv.slice(2)); +const targetDirArg = argv.target_dir +const condaEnvLibraryDirArg = argv.conda_env_library_dir - console.error('Usage: fetch-python-package-resources.js '); +if (!targetDirArg || !condaEnvLibraryDirArg) { + console.error('Usage: node fetch-python-package-resources.js --target_dir=$DIR --conda_env_library_dir=$DIR\n'); process.exit(1); } -const targetDir = path.resolve(process.argv[2]); +const targetDir = path.resolve(targetDirArg); +const condaTargetDir = path.join(targetDir, 'conda-env-lib') +const condaEnvLibraryDir = path.resolve(condaEnvLibraryDirArg); const embeddablePythonUrl = 'https://raw.githubusercontent.com/adang1345/PythonWindows/master/3.11.10/python-3.11.10-embed-amd64.zip'; const getPipScriptUrl = 'https://bootstrap.pypa.io/get-pip.py' const sevenZrExeUrl = 'https://www.7-zip.org/a/7zr.exe' -function fetchFileIfNotPresent(url, targetDir) { +function fetchFileIfNotPresent(url) { const expectedFilePath = path.join(targetDir, getBaseFileName(url)) if (fs.existsSync(expectedFilePath)) { console.log(`omitting fetching of ${url} as ${expectedFilePath} already exists`) } else { - fetchFile(url, targetDir) + fetchFile(url) } } -function fetchFile(url, targetDir) { +function fetchFile(url) { https.get(url, (response) => { const filePath = path.join(targetDir, getBaseFileName(url)) const file = fs.createWriteStream(filePath); @@ -48,22 +53,38 @@ function getBaseFileName(url) { } function prepareTargetPath() { - if (!fs.existsSync(targetDir)) { + if (!fs.existsSync(condaTargetDir)) { fs.mkdirSync(targetDir, { recursive: true }); } } -function provideLibuvDlls() { - console.error("provideLibuvDlls is currently only mocked") - console.error(`please simlink to conda virtual env into ${targetDir} manually`) +function copyLibuvDllsIfNotPresent() { + if (fs.existsSync(path.join(condaTargetDir, 'Library', 'bin', 'uv.dll'))) { + console.log(`omitting fetching copying of libuvDLLs, as they already exist`) + } else { + if (!path.join(condaEnvLibraryDir, 'bin', 'uv.dll')) { + console.log(`provided conda env at ${condaEnvLibraryDir} is missing uv.dll. Aborting`) + process.exit(1); + } + fs.cp(condaEnvLibraryDir, path.join(condaTargetDir, 'Library'), { recursive: true }, (err) => { + if (err) { + console.error(err); + console.log('Failed to copy directory'); + process.exit(1) + } else { + console.log('Directory copied successfully'); + } + }); + } } + function main() { prepareTargetPath() - fetchFileIfNotPresent(embeddablePythonUrl, targetDir) - fetchFileIfNotPresent(getPipScriptUrl, targetDir) - fetchFileIfNotPresent(sevenZrExeUrl, targetDir) - provideLibuvDlls() + fetchFileIfNotPresent(embeddablePythonUrl) + fetchFileIfNotPresent(getPipScriptUrl) + fetchFileIfNotPresent(sevenZrExeUrl) + copyLibuvDllsIfNotPresent() } main() diff --git a/WebUI/build/scripts/install-full-python-env.js b/WebUI/build/scripts/install-full-python-env.js new file mode 100644 index 00000000..3990d2a7 --- /dev/null +++ b/WebUI/build/scripts/install-full-python-env.js @@ -0,0 +1,73 @@ +// Usage: node pack-offline.js +// +const fs = require('fs'); +const path = require('path'); +const AdmZip = require('adm-zip'); +const childProcess = require('child_process'); + +const argv = require('minimist')(process.argv.slice(2)); +const envDirArg = argv.env_dir +const platformArg = argv.platform + +if (!envDirArg || !platformArg) { + console.error('Usage: node install-full-python-env.js --env_dir=$DIR ---platform=arc|ultra|ultra2\n'); + process.exit(1); +} + +const envDir = existingFileOrExit(path.resolve(envDirArg)); +const platform = platformArg; + +function existingFileOrExit(filePath) { + if (!fs.existsSync(filePath)) { + console.error('Resource not found:', filePath); + process.exit(1); + } + return filePath +} + + +function installPip(pythonExe, getPipFilePath) { + const runGetPip = childProcess.spawnSync(pythonExe, [getPipFilePath]); + console.log(runGetPip.stdout.toString()); + //console.error(runGetPip.stderr.toString()); + if (runGetPip.status!== 0) { + console.error('Failed to install requirements'); + process.exit(1); + } + console.log('Successfully installed pip'); +} + +function runPipInstall(pythonExe, requirementsFilePath) { + console.log(`installing python dependencies from ${requirementsFilePath}`); + const pipInstall = childProcess.spawnSync(pythonExe, ['-m', 'pip', 'install', '-r', requirementsFilePath]); + console.log(pipInstall.stdout.toString()); + console.error(pipInstall.stderr.toString()); + if (pipInstall.status!== 0) { + console.error('Failed to install requirements'); + process.exit(1); + } + console.log(`Installed dependencies from ${requirementsFilePath} successfully`); +} + +function copyToTargetDir(sourceDir, targetDir) { + fs.cpSync(sourceDir, targetDir, { recursive: true }); + console.log(`copied resources to ${targetDir}`) +} + + +function main() { + const targetDir = path.join(envDir, '..', `env-full-${platform}`); + copyToTargetDir(envDir, targetDir) + + const pythonExe = existingFileOrExit(path.join(targetDir, 'python.exe')); + const getPipFile = existingFileOrExit(path.join(targetDir, 'get-pip.py')); + + const platformSpecificRequirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..','..', 'service', `requirements-${platform}.txt`)); + const requirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', '..', 'service', `requirements.txt`)); + + installPip(pythonExe, getPipFile) + runPipInstall(pythonExe, platformSpecificRequirementsTxt) + runPipInstall(pythonExe, requirementsTxt) +} + +main(); diff --git a/WebUI/build/scripts/pack-offline.js b/WebUI/build/scripts/pack-offline.js deleted file mode 100644 index e2c4e6a9..00000000 --- a/WebUI/build/scripts/pack-offline.js +++ /dev/null @@ -1,111 +0,0 @@ -// Usage: node pack-offline.js -// -const fs = require('fs'); -const path = require('path'); -const AdmZip = require('adm-zip'); -const childProcess = require('child_process'); - -if (process.argv.length < 4) { - console.error('Usage: node pack-offline.js '); - process.exit(1); -} - -const packageResDir = existingFileOrExit(path.resolve(process.argv[2])); -const platform = process.argv[3]; - -const sevenZipExe = existingFileOrExit(path.join(packageResDir, '7zr.exe')); -const zippedPyenv = existingFileOrExit(path.join(packageResDir, 'env.7z')); - -function existingFileOrExit(filePath) { - if (!fs.existsSync(filePath)) { - console.error('Resource not found:', filePath); - process.exit(1); - } - return filePath -} - - -function unzippedPackagedPyenv(pyenvArchive) { - existingFileOrExit(pyenvArchive) - const pyenvTargetDir = workDir - - if (fs.existsSync(pyenvTargetDir)) { - console.warn("Removing existing offline env directory:", pyenvTargetDir); - fs.rmSync(workDir, { recursive: true }); - } - - const unzip = childProcess.spawnSync(sevenZipExe, ['x', pyenvArchive, `-o${workDir}`]); - console.log(unzip.stdout.toString()); - console.error(unzip.stderr.toString()); - if (unzip.status !== 0) { - console.error('Failed to extract env.7z'); - process.exit(1); - } - - const offlineEnvDir = existingFileOrExit(path.join(workDir, 'env')); - return offlineEnvDir -} - -function installPip(getPipFilePath, workingDir) { - const runGetPip = spawn(pythonExe, [getPipFilePath], { cwd: workingDir }); - runGetPip.stdout.on('data', (data) => { - console.log(data.toString()); - }); - runGetPip.stderr.on('data', (data) => { - console.error(data.toString()); - }); - pipInstall.on('close', (code) => { - if (code !== 0) { - console.error('Failed to install pip'); - process.exit(1); - } - }); -} - -function runPipInstall(requirementsFilePath, workingDir) { - const pipInstall = spawn(pythonExe, ['-m', 'pip', 'install', '-r', requirementsFilePath], { cwd: workingDir }); - pipInstall.stdout.on('data', (data) => { - console.log(data.toString()); - }); - pipInstall.stderr.on('data', (data) => { - console.error(data.toString()); - }); - pipInstall.on('close', (code) => { - if (code !== 0) { - console.error('Failed to install requirements'); - process.exit(1); - } - }); -} - -function zipPyenv(pyEnvDir) { - const offlineEnvArchiveTargetPath = path.join(packageResDir, `env-offline-${platform}.7z`); - const zip = childProcess.spawnSync((sevenZipExe, ['a', offlineEnvArchiveTargetPath, existingFileOrExit(pyEnvDir)]); - console.log(zip.stdout.toString()); - console.error(zip.stderr.toString()); - if (zip.status !== 0) { - console.error('Failed to compress offline env directory'); - process.exit(1); - } - - console.log('Offline env has been created successfully:', offlineEnvArchiveTargetPath); -} - -function main() { - const workDir = path.join(__dirname, `env-${platform}`); - - const offlineEnvDir = unzippedPackagedPyenv(zippedPyenv) - const pythonExe = existingFileOrExit(path.join(offlineEnvDir, 'python.exe')); - - const getPipFile = existingFileOrExit(path.join(offlineEnvDir, 'get-pip.py')); - const platformSpecificRequirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', 'service', `requirements-${platform}.txt`)); - const requirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', 'service', `requirements.txt`)); - const comfyUiRequirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', 'ComfyUI', `requirements.txt`)); - const requirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', 'ComfyUI', 'custom_nodes', 'ComfyUI-GGUF', `requirements.txt`)); - - installPip(getPipFile, offlineEnvDir) - runPipInstall(platformSpecificRequirementsTxt, offlineEnvDir) - runPipInstall(requirementsTxt, offlineEnvDir) - - zipPyenv(offlineEnvDir) -} \ No newline at end of file diff --git a/WebUI/build/scripts/prebuild.js b/WebUI/build/scripts/prebuild.js deleted file mode 100644 index 55e3c414..00000000 --- a/WebUI/build/scripts/prebuild.js +++ /dev/null @@ -1,63 +0,0 @@ -// Usage: node prebuild.js - -const fs = require('fs'); -const path = require('path'); - -if (process.argv.length < 5) { - console.error('Usage: node prebuild.js '); - process.exit(1); -} - -const packageResDir = path.resolve(process.argv[2]); -const serviceDir = path.resolve(process.argv[3]); -const buildDir = path.resolve(process.argv[4]); -const comfyUIDir = path.join(serviceDir, '..', 'ComfyUI') - -if (!fs.existsSync(packageResDir)) { - console.error('Directory not found:', packageResDir); - process.exit(1); -} - -// remove link if already exists -const serviceLink = path.join(buildDir, 'service'); -if (fs.existsSync(serviceLink)) { - fs.unlinkSync(serviceLink); - console.log('Removed symlink:', serviceLink); -} -// make link from service to /service -fs.symlinkSync(serviceDir, serviceLink, 'junction'); -console.log('Created symlink:', serviceDir); - -// remove link if already exists -const comfyUILink = path.join(buildDir, 'ComfyUI'); -if (fs.existsSync(comfyUILink)) { - fs.unlinkSync(comfyUILink); - console.log('Removed symlink:', comfyUILink); -} -fs.symlinkSync(comfyUIDir, comfyUILink, 'junction'); -console.log('Created symlink:', comfyUIDir); - - - -// Copy all files under package_res_dir to build_dir -const files = fs.readdirSync(packageResDir); -for (const file of files) { - const src = path.join(packageResDir, file); - const dest = path.join(buildDir, file); - fs.copyFileSync(src, dest); - console.log('Copied:', src, 'to:', dest); -} - -// check 7zr.exe exists -const sevenZipExe = path.join(buildDir, '7zr.exe'); -if (!fs.existsSync(sevenZipExe)) { - console.error('7zr.exe not found:', sevenZipExe); - process.exit(1); -} - -// check there's at least one .7z file in build_dir -const archives = fs.readdirSync(buildDir).filter(f => f.endsWith('.7z')); -if (archives.length === 0) { - console.error('No .7z file found in:', buildDir); - process.exit(1); -} diff --git a/WebUI/build/scripts/pack-python.js b/WebUI/build/scripts/prepare-python-env.js similarity index 74% rename from WebUI/build/scripts/pack-python.js rename to WebUI/build/scripts/prepare-python-env.js index 4e9ff8e9..14b7e629 100644 --- a/WebUI/build/scripts/pack-python.js +++ b/WebUI/build/scripts/prepare-python-env.js @@ -1,26 +1,31 @@ -// Usage: node pack-python.js +// Usage: node prepare-python-env.js --target_dir=$DIR ---env_resources_dir=$DIR const fs = require('fs'); const path = require('path'); const AdmZip = require('adm-zip'); const childProcess = require('child_process'); -if (process.argv.length < 3) { - console.error('Usage: node pack-python.js '); +const argv = require('minimist')(process.argv.slice(2)); +const buildResourcesDirArg = argv.build_resources_dir +const targetDirArg = argv.target_dir + +if (!buildResourcesDirArg || !targetDirArg) { + console.error('Usage: node prepare-python-env.js --target_dir=$DIR ---env_resources_dir=$DIR\n'); process.exit(1); } -const pythonPackageResourcesDir = path.resolve(process.argv[2]); -const targetResDir = path.resolve(process.argv[3]); + +const envResourcesDir = path.resolve(buildResourcesDirArg); +const targetDir = path.resolve(targetDirArg); const webUIBuildDir = path.join(__dirname, '..', '..'); const pythonEmbedDir = path.join(webUIBuildDir, '..', 'env'); -const packageResourceFiles = fs.readdirSync(pythonPackageResourcesDir); -const pythonEmbedZipFile = path.join(pythonPackageResourcesDir, packageResourceFiles.find((fileName) => { return fileName.startsWith('python') && fileName.endsWith('.zip') })); -const condaDir = path.join(pythonPackageResourcesDir, packageResourceFiles.find((fileName) => { return fileName.includes('conda') })); +const envResourcesFiles = fs.readdirSync(envResourcesDir); +const pythonEmbedZipFile = path.join(envResourcesDir, envResourcesFiles.find((fileName) => { return fileName.startsWith('python') && fileName.endsWith('.zip') })); +const condaDir = path.join(envResourcesDir, envResourcesFiles.find((fileName) => { return fileName.includes('conda') })); const condaBinDir = path.join(condaDir, 'Library', 'bin'); -const getPipFile = path.join(pythonPackageResourcesDir, 'get-pip.py'); -const sevenZipExe = path.join(pythonPackageResourcesDir, '7zr.exe') +const getPipFile = path.join(envResourcesDir, 'get-pip.py'); +const sevenZipExe = path.join(envResourcesDir, '7zr.exe') const sevenZipBinary = () => { if (fs.existsSync(sevenZipExe)) { return sevenZipExe; @@ -55,9 +60,9 @@ function verifyFilesExist() { } console.log('all required files exist.') - if (!fs.existsSync(targetResDir)) { - console.log(`Creating missing target dir: ${targetResDir}`) - fs.mkdirSync(targetResDir, { recursive: true }); + if (!fs.existsSync(targetDir)) { + console.log(`Creating missing target dir: ${targetDir}`) + fs.mkdirSync(targetDir, { recursive: true }); } } @@ -83,7 +88,6 @@ function createPythonEnvFromEmbedabblePythonZip() { python311.zip . ../service -../ComfyUI # Uncomment to run site.main() automatically import site @@ -131,9 +135,9 @@ function compressPythonEnvDirectory(pyEnvDirPath) { function copyToTargetDir(filePaths) { for (const sourceFilePath of filePaths) { - const destinationPath = path.join(targetResDir, path.basename(sourceFilePath)); + const destinationPath = path.join(targetDir, path.basename(sourceFilePath)); fs.copyFileSync(sourceFilePath, destinationPath); - console.log(`copied ${path.basename(sourceFilePath)} into ${targetResDir}`) + console.log(`copied ${path.basename(sourceFilePath)} into ${targetDir}`) } } @@ -141,8 +145,8 @@ function main() { verifyFilesExist(); const pyEnvPath = createPythonEnvFromEmbedabblePythonZip(); patchCondaDllsIntoPythonEnv(pyEnvPath); - const sevenZippedPyenv = compressPythonEnvDirectory(pyEnvPath); - copyToTargetDir([sevenZippedPyenv, sevenZipExe]); + //const sevenZippedPyenv = compressPythonEnvDirectory(pyEnvPath); + //copyToTargetDir([sevenZippedPyenv, sevenZipExe]); } main(); diff --git a/WebUI/build/scripts/provide-electron-build-resources.js b/WebUI/build/scripts/provide-electron-build-resources.js new file mode 100644 index 00000000..0054d48d --- /dev/null +++ b/WebUI/build/scripts/provide-electron-build-resources.js @@ -0,0 +1,69 @@ +// Usage: node provide-build-resources.js --python_env_dir=$DIR ---backend_dir=$DIR --target_dir=$DIR + +const fs = require('fs'); +const path = require('path'); +const childProcess = require('child_process'); + +const argv = require('minimist')(process.argv.slice(2)); +const buildResourcesDirArg = argv.build_resources_dir +const pythonEnvDirArg = argv.python_env_dir +const backendDirArg = argv.backend_dir +const targetDirectoryArg = argv.target_dir + + +if (!buildResourcesDirArg || !pythonEnvDirArg || !backendDirArg || !targetDirectoryArg) { + console.error('Usage: node prebuild.js --build_resources_dir=$DIR --python_env_dir=$DIR ---backend_dir=$DIR --target_dir=$DIR\n'); + process.exit(1); +} + +const buildResourcesDir = path.resolve(buildResourcesDirArg) +const pythenEnvDir = path.resolve(pythonEnvDirArg) +const backendDir = path.resolve(backendDirArg) +const targetDir = path.resolve(targetDirectoryArg) + + +function symlinkBackendDir(backendDir, serviceLink) { + // remove link if already exists + if (fs.existsSync(serviceLink)) { + fs.unlinkSync(serviceLink); + console.log('Removed symlink:', serviceLink); + } + // make link from service to /service + fs.symlinkSync(backendDir, serviceLink, 'junction'); + console.log('Created symlink:', backendDir); +} + + +function zipPythonEnv(sevenZipExe, pythonEnvDir, targetDir) { + const envArchiveTargetPath = path.join(targetDir, `env.7z`); + console.log(`zipping offline env to ${envArchiveTargetPath}`); + + const zip = childProcess.spawnSync(sevenZipExe, ['a', envArchiveTargetPath, pythenEnvDir]); + console.log(zip.stdout.toString()); + console.error(zip.stderr.toString()); + if (zip.status !== 0) { + console.error('Failed to compress offline env directory'); + process.exit(1); + } + + console.log('Offline env has been created successfully:', envArchiveTargetPath); +} + +function copyResources(targetDir, ...files) { + for (const file of files) { + fs.copyFileSync(src, path.join(targetDir, path.basename(filePath))); + console.log('Copied:', src, 'to:', dest); + } +} + +function main() { + const sevenZipExe = path.join(buildResourcesDir, '7zr.exe'); + + zipPythonEnv(sevenZipExe, pythenEnvDir, path.join(targetDir, `env.7z`)); + symlinkBackendDir(backendDir, path.join(targetDir, 'service')) + copyResources(targetDir, + sevenZipExe + ) +} + +main() diff --git a/WebUI/build/scripts/render-template.js b/WebUI/build/scripts/render-template.js index bd328932..b51e3123 100644 --- a/WebUI/build/scripts/render-template.js +++ b/WebUI/build/scripts/render-template.js @@ -1,38 +1,57 @@ const fs = require('fs'); const path = require('path'); const buildDir = path.join(__dirname, '..') + +const argv = require('minimist')(process.argv.slice(2)); +const onlineInstallerFlag = argv.online_installer +const platform = argv.platform + +if (!platform) { + console.error(`Usage: node render-template.js --platform=$PLATFORM --online_installer (or --no-online_installer)\n`); + process.exit(1); +} + +console.log(onlineInstallerFlag) + // Function to render the template with environment variables function renderTemplate(templatePath, outputPath, variables) { - // Read the template content - fs.readFile(templatePath, 'utf8', (err, data) => { - if (err) { - console.error(`Error reading template: ${err}`); - return; - } + // Read the template content + fs.readFile(templatePath, 'utf8', (err, data) => { + if (err) { + console.error(`Error reading template: ${err}`); + return; + } - // Replace placeholders with environment variable values - let renderedData = data; - for (const [key, value] of Object.entries(variables)) { - const placeholder = new RegExp(`\\$\\{${key}\\}`, 'g'); - renderedData = renderedData.replace(placeholder, value); - } + // Replace placeholders with environment variable values + let renderedData = data; + for (const [key, value] of Object.entries(variables)) { + const placeholder = new RegExp(`\\$\\{${key}\\}`, 'g'); + renderedData = renderedData.replace(placeholder, value); + } - // Write the rendered content to the output file - fs.writeFile(outputPath, renderedData, 'utf8', (err) => { - if (err) { - console.error(`Error writing output file: ${err}`); - return; - } - console.log(`Template has been rendered successfully.`); + // Write the rendered content to the output file + fs.writeFile(outputPath, renderedData, 'utf8', (err) => { + if (err) { + console.error(`Error writing output file: ${err}`); + return; + } + console.log(`Template has been rendered successfully.`); + }); }); - }); } -// Example usage -const templatePath = path.join(buildDir, 'installer.nsh.template'); +function installerTemplate() { + if (onlineInstallerFlag) { + return 'installer.nsh.template' + } else { + return 'installer-offline.nsh.template' + } +} + +const templatePath = path.join(buildDir, installerTemplate()); const outputPath = path.join(buildDir, 'installer.nsh'); const variables = { - PLATFORM: process.env.PLATFORM || 'arc' + PLATFORM: platform }; renderTemplate(templatePath, outputPath, variables); diff --git a/WebUI/build/ultra_before_build.bat b/WebUI/build/ultra_before_build.bat deleted file mode 100644 index e2e3d2c3..00000000 --- a/WebUI/build/ultra_before_build.bat +++ /dev/null @@ -1,19 +0,0 @@ -@echo off -if EXIST ".\external\service" ( - rd /s /q ".\external\service" -) -mklink /J ".\external\service" "..\service" - -if NOT EXIST "..\package_res\ultra_env.7z" ( - setlocal enabledelayedexpansion - pushd "..\package_res" - set "absolutePath=!cd!" - echo "You must place ultra_env.7z under the directory !absolutePath!" - popd - endlocal - Exit -1 -) -IF EXIST ".\external\env.7z" ( - del /F /Q ".\external\env.7z" -) -mklink /H ".\external\env.7z" "..\package_res\ultra_env.7z" \ No newline at end of file diff --git a/WebUI/package.json b/WebUI/package.json index ea866814..8567cb30 100644 --- a/WebUI/package.json +++ b/WebUI/package.json @@ -4,16 +4,21 @@ "version": "1.22.1-beta", "scripts": { "dev": "cross-env VITE_PLATFORM_TITLE=\"for Local® Dev™ Mode\" vite", - "fetch-build-resources": "cross-env node ./build/scripts/fetch-python-package-resources.js ../python_package_res", - "pack-offline": "cross-env node build/scripts/pack-offline.js ./package_res", - "pack-python": "cross-env node build/scripts/pack-python.js ../python_package_res ./npm_package_res", - "prebuild": "cross-env node build/scripts/prebuild.js ./npm_package_res ../service ./external", - "build:all": "cross-env npm run prebuild && npm run build:arc && npm run build:ultra && npm run build:ultra2", - "build:arc": "cross-env-shell PLATFORM=\"arc\" VITE_PLATFORM_TITLE=\"for Intel® Arc™\" \"vue-tsc && vite build && node ./build/scripts/render-template.js && electron-builder --config build/build-config.json --win --x64\"", - "build:ultra": "cross-env-shell PLATFORM=\"ultra\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra\" \"vue-tsc && vite build && node ./build/scripts/render-template.js && electron-builder --config build/build-config.json --win --x64\"", - "build:ultra2": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"vue-tsc && vite build && node ./build/scripts/render-template.js && electron-builder --config build/build-config.json --win --x64\"", + "preview": "vite preview", + + "fetch-build-resources": "cross-env node ./build/scripts/fetch-python-package-resources.js --target_dir=../build_resources", + "prepare-python-env": "cross-env node ./build/scripts/prepare-python-env.js --build_resources_dir=../build_resources --target_dir=../env", + "install-full-python-env": "cross-env node ./build/scripts/install-full-python-env.js --env_dir=../env", + "provide-electron-build-resources": "cross-env node build/scripts/provide-electron-build-resources.js --build_resources_dir=../build_resources --backend_dir=../service --target_dir=./external", + + "build:arc": "cross-env-shell PLATFORM=\"arc\" VITE_PLATFORM_TITLE=\"for Intel® Arc™\" \"vue-tsc && vite build && electron-builder --config build/build-config.json --win --x64\"", + "build:arc-offline": "cross-env-shell PLATFORM=\"arc\" VITE_PLATFORM_TITLE=\"for Intel® Arc™\" \"vue-tsc && vite build && electron-builder --config build/build-config-offline.json --win --x64\"", + + "build:ultra": "cross-env-shell PLATFORM=\"ultra\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra\" \"vue-tsc && vite build && electron-builder --config build/build-config.json --win --x64\"", + "build:ultra-offline": "cross-env-shell PLATFORM=\"ultra\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra\" \"vue-tsc && vite build && electron-builder --config build/build-config-offline.json --win --x64\"", + + "build:ultra2": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"vue-tsc && vite build && electron-builder --config build/build-config.json --win --x64\"", "build:ultra2-offline": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"npm run pack-offline ultra2 && npm run prebuild && vue-tsc && vite build && electron-builder --config build/build-config-offline.json --win --x64\"", - "preview": "vite preview" }, "dependencies": { "@radix-icons/vue": "^1.0.0", diff --git a/service/requirements-arc.txt b/service/requirements-arc.txt index 2720f6c8..5c2239ec 100644 --- a/service/requirements-arc.txt +++ b/service/requirements-arc.txt @@ -13,4 +13,5 @@ ipex_llm==2.2.0b2 bigdl-core-xe-23==2.6.0b2 bigdl-core-xe-addons-23==2.6.0b2 bigdl-core-xe-batch-23==2.6.0b2 -onednn-devel==2024.1.1 \ No newline at end of file +onednn-devel==2024.1.1 +# C:\intel_build_tests\ipex23_wheels\intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl