From 6f1dcf385ba2403c020c7925b92bafdb2ff7be82 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Thu, 20 Feb 2025 20:18:56 +0000 Subject: [PATCH 1/8] Move find_port() util to its own file. --- invokeai/app/api_app.py | 20 +++++--------------- invokeai/app/util/startup_utils.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 15 deletions(-) create mode 100644 invokeai/app/util/startup_utils.py diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index 9cfebbdca17..b2220051455 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -1,7 +1,6 @@ import asyncio import logging import mimetypes -import socket from contextlib import asynccontextmanager from pathlib import Path @@ -17,8 +16,6 @@ from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from torch.backends.mps import is_available as is_mps_available -# for PyCharm: -# noinspection PyUnresolvedReferences import invokeai.backend.util.hotfixes # noqa: F401 (monkeypatching on import) import invokeai.frontend.web as web_dir from invokeai.app.api.dependencies import ApiDependencies @@ -39,6 +36,10 @@ from invokeai.app.invocations.load_custom_nodes import load_custom_nodes from invokeai.app.services.config.config_default import get_config from invokeai.app.util.custom_openapi import get_openapi_func + +# for PyCharm: +# noinspection PyUnresolvedReferences +from invokeai.app.util.startup_utils import find_open_port from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.logging import InvokeAILogger @@ -211,17 +212,6 @@ def check_cudnn(logger: logging.Logger) -> None: def invoke_api() -> None: - def find_port(port: int) -> int: - """Find a port not in use starting at given port""" - # Taken from https://waylonwalker.com/python-find-available-port/, thanks Waylon! - # https://github.com/WaylonWalker - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(1) - if s.connect_ex(("localhost", port)) == 0: - return find_port(port=port + 1) - else: - return port - if app_config.dev_reload: try: import jurigged @@ -234,7 +224,7 @@ def find_port(port: int) -> int: jurigged.watch(logger=InvokeAILogger.get_logger(name="jurigged").info) global port - port = find_port(app_config.port) + port = find_open_port(app_config.port) if port != app_config.port: logger.warn(f"Port {app_config.port} in use, using port {port}") diff --git a/invokeai/app/util/startup_utils.py b/invokeai/app/util/startup_utils.py new file mode 100644 index 00000000000..93adc236698 --- /dev/null +++ b/invokeai/app/util/startup_utils.py @@ -0,0 +1,13 @@ +import socket + + +def find_open_port(port: int) -> int: + """Find a port not in use starting at given port""" + # Taken from https://waylonwalker.com/python-find-available-port/, thanks Waylon! + # https://github.com/WaylonWalker + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(1) + if s.connect_ex(("localhost", port)) == 0: + return find_open_port(port=port + 1) + else: + return port From 35910d3952c16d56788e98bb12ec7603d7d969ba Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Thu, 20 Feb 2025 20:35:14 +0000 Subject: [PATCH 2/8] Move check_cudnn() and jurigged setup to startup_utils.py. --- invokeai/app/api_app.py | 29 ++------------------------ invokeai/app/util/startup_utils.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index b2220051455..f1e72c8a66b 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -4,7 +4,6 @@ from contextlib import asynccontextmanager from pathlib import Path -import torch import uvicorn from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -39,7 +38,7 @@ # for PyCharm: # noinspection PyUnresolvedReferences -from invokeai.app.util.startup_utils import find_open_port +from invokeai.app.util.startup_utils import check_cudnn, enable_dev_reload, find_open_port from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.logging import InvokeAILogger @@ -195,33 +194,9 @@ def overridden_redoc() -> HTMLResponse: ) # docs favicon is in here -def check_cudnn(logger: logging.Logger) -> None: - """Check for cuDNN issues that could be causing degraded performance.""" - if torch.backends.cudnn.is_available(): - try: - # Note: At the time of writing (torch 2.2.1), torch.backends.cudnn.version() only raises an error the first - # time it is called. Subsequent calls will return the version number without complaining about a mismatch. - cudnn_version = torch.backends.cudnn.version() - logger.info(f"cuDNN version: {cudnn_version}") - except RuntimeError as e: - logger.warning( - "Encountered a cuDNN version issue. This may result in degraded performance. This issue is usually " - "caused by an incompatible cuDNN version installed in your python environment, or on the host " - f"system. Full error message:\n{e}" - ) - - def invoke_api() -> None: if app_config.dev_reload: - try: - import jurigged - except ImportError as e: - logger.error( - 'Can\'t start `--dev_reload` because jurigged is not found; `pip install -e ".[dev]"` to include development dependencies.', - exc_info=e, - ) - else: - jurigged.watch(logger=InvokeAILogger.get_logger(name="jurigged").info) + enable_dev_reload() global port port = find_open_port(app_config.port) diff --git a/invokeai/app/util/startup_utils.py b/invokeai/app/util/startup_utils.py index 93adc236698..7d902888c1e 100644 --- a/invokeai/app/util/startup_utils.py +++ b/invokeai/app/util/startup_utils.py @@ -1,5 +1,10 @@ +import logging import socket +import torch + +from invokeai.backend.util.logging import InvokeAILogger + def find_open_port(port: int) -> int: """Find a port not in use starting at given port""" @@ -11,3 +16,31 @@ def find_open_port(port: int) -> int: return find_open_port(port=port + 1) else: return port + + +def check_cudnn(logger: logging.Logger) -> None: + """Check for cuDNN issues that could be causing degraded performance.""" + if torch.backends.cudnn.is_available(): + try: + # Note: At the time of writing (torch 2.2.1), torch.backends.cudnn.version() only raises an error the first + # time it is called. Subsequent calls will return the version number without complaining about a mismatch. + cudnn_version = torch.backends.cudnn.version() + logger.info(f"cuDNN version: {cudnn_version}") + except RuntimeError as e: + logger.warning( + "Encountered a cuDNN version issue. This may result in degraded performance. This issue is usually " + "caused by an incompatible cuDNN version installed in your python environment, or on the host " + f"system. Full error message:\n{e}" + ) + + +def enable_dev_reload() -> None: + """Enable hot reloading on python file changes during development.""" + try: + import jurigged + except ImportError as e: + raise RuntimeError( + 'Can\'t start `--dev_reload` because jurigged is not found; `pip install -e ".[dev]"` to include development dependencies.' + ) from e + else: + jurigged.watch(logger=InvokeAILogger.get_logger(name="jurigged").info) From ca23b5337e7747a7fbfdf3903f00a0ea3720a2f5 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Thu, 20 Feb 2025 20:50:22 +0000 Subject: [PATCH 3/8] Simplify port selection logic to avoid the need for a global port variable. --- invokeai/app/api_app.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index f1e72c8a66b..d29356985b6 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -60,10 +60,6 @@ loop = asyncio.new_event_loop() -# We may change the port if the default is in use, this global variable is used to store the port so that we can log -# the correct port when the server starts in the lifespan handler. -port = app_config.port - # Load custom nodes. This must be done after importing the Graph class, which itself imports all modules from the # invocations module. The ordering here is implicit, but important - we want to load custom nodes after all the # core nodes have been imported so that we can catch when a custom node clobbers a core node. @@ -77,7 +73,7 @@ async def lifespan(app: FastAPI): # Log the server address when it starts - in case the network log level is not high enough to see the startup log proto = "https" if app_config.ssl_certfile else "http" - msg = f"Invoke running on {proto}://{app_config.host}:{port} (Press CTRL+C to quit)" + msg = f"Invoke running on {proto}://{app_config.host}:{app_config.port} (Press CTRL+C to quit)" # Logging this way ignores the logger's log level and _always_ logs the message record = logger.makeRecord( @@ -198,17 +194,17 @@ def invoke_api() -> None: if app_config.dev_reload: enable_dev_reload() - global port - port = find_open_port(app_config.port) - if port != app_config.port: - logger.warn(f"Port {app_config.port} in use, using port {port}") + orig_config_port = app_config.port + app_config.port = find_open_port(app_config.port) + if orig_config_port != app_config.port: + logger.warning(f"Port {orig_config_port} is already in use. Using port {app_config.port}.") check_cudnn(logger) config = uvicorn.Config( app=app, host=app_config.host, - port=port, + port=app_config.port, loop="asyncio", log_level=app_config.log_level_network, ssl_certfile=app_config.ssl_certfile, @@ -223,7 +219,3 @@ def invoke_api() -> None: uvicorn_logger.addHandler(hdlr) loop.run_until_complete(server.serve()) - - -if __name__ == "__main__": - invoke_api() From f345c0fabc05a38a8596cd011b50924bd3ec35a8 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Fri, 21 Feb 2025 15:16:40 +0000 Subject: [PATCH 4/8] Create an apply_monkeypatches() start util. --- invokeai/app/api_app.py | 8 ++------ invokeai/app/util/startup_utils.py | 9 +++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index d29356985b6..a5bc5ec8ca0 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -13,9 +13,7 @@ from fastapi_events.handlers.local import local_handler from fastapi_events.middleware import EventHandlerASGIMiddleware from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint -from torch.backends.mps import is_available as is_mps_available -import invokeai.backend.util.hotfixes # noqa: F401 (monkeypatching on import) import invokeai.frontend.web as web_dir from invokeai.app.api.dependencies import ApiDependencies from invokeai.app.api.no_cache_staticfiles import NoCacheStaticFiles @@ -38,15 +36,13 @@ # for PyCharm: # noinspection PyUnresolvedReferences -from invokeai.app.util.startup_utils import check_cudnn, enable_dev_reload, find_open_port +from invokeai.app.util.startup_utils import apply_monkeypatches, check_cudnn, enable_dev_reload, find_open_port from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.logging import InvokeAILogger app_config = get_config() - -if is_mps_available(): - import invokeai.backend.util.mps_fixes # noqa: F401 (monkeypatching on import) +apply_monkeypatches() logger = InvokeAILogger.get_logger(config=app_config) diff --git a/invokeai/app/util/startup_utils.py b/invokeai/app/util/startup_utils.py index 7d902888c1e..7e1d2b2dc07 100644 --- a/invokeai/app/util/startup_utils.py +++ b/invokeai/app/util/startup_utils.py @@ -44,3 +44,12 @@ def enable_dev_reload() -> None: ) from e else: jurigged.watch(logger=InvokeAILogger.get_logger(name="jurigged").info) + + +def apply_monkeypatches() -> None: + """Apply monkeypatches to fix issues with third-party libraries.""" + + import invokeai.backend.util.hotfixes # noqa: F401 (monkeypatching on import) + + if torch.backends.mps.is_available(): + import invokeai.backend.util.mps_fixes # noqa: F401 (monkeypatching on import) From 38991ffc35c5758c5712e35f50799ae591e3556b Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Fri, 21 Feb 2025 15:21:48 +0000 Subject: [PATCH 5/8] Add register_mime_types() startup util. --- invokeai/app/api_app.py | 18 ++++++++---------- invokeai/app/util/startup_utils.py | 9 +++++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index a5bc5ec8ca0..f97abd18024 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -1,6 +1,5 @@ import asyncio import logging -import mimetypes from contextlib import asynccontextmanager from pathlib import Path @@ -33,23 +32,22 @@ from invokeai.app.invocations.load_custom_nodes import load_custom_nodes from invokeai.app.services.config.config_default import get_config from invokeai.app.util.custom_openapi import get_openapi_func - -# for PyCharm: -# noinspection PyUnresolvedReferences -from invokeai.app.util.startup_utils import apply_monkeypatches, check_cudnn, enable_dev_reload, find_open_port +from invokeai.app.util.startup_utils import ( + apply_monkeypatches, + check_cudnn, + enable_dev_reload, + find_open_port, + register_mime_types, +) from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.logging import InvokeAILogger app_config = get_config() apply_monkeypatches() - +register_mime_types() logger = InvokeAILogger.get_logger(config=app_config) -# fix for windows mimetypes registry entries being borked -# see https://github.com/invoke-ai/InvokeAI/discussions/3684#discussioncomment-6391352 -mimetypes.add_type("application/javascript", ".js") -mimetypes.add_type("text/css", ".css") torch_device_name = TorchDevice.get_torch_device_name() logger.info(f"Using torch device: {torch_device_name}") diff --git a/invokeai/app/util/startup_utils.py b/invokeai/app/util/startup_utils.py index 7e1d2b2dc07..9c3f194c283 100644 --- a/invokeai/app/util/startup_utils.py +++ b/invokeai/app/util/startup_utils.py @@ -1,4 +1,5 @@ import logging +import mimetypes import socket import torch @@ -53,3 +54,11 @@ def apply_monkeypatches() -> None: if torch.backends.mps.is_available(): import invokeai.backend.util.mps_fixes # noqa: F401 (monkeypatching on import) + + +def register_mime_types() -> None: + """Register additional mime types for windows.""" + # Fix for windows mimetypes registry entries being borked. + # see https://github.com/invoke-ai/InvokeAI/discussions/3684#discussioncomment-6391352 + mimetypes.add_type("application/javascript", ".js") + mimetypes.add_type("text/css", ".css") From 68d14de3ee00afca2c27a10981000955198985a1 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Fri, 21 Feb 2025 16:26:42 +0000 Subject: [PATCH 6/8] Split run_app.py and api_app.py so that api_app.py is more narrowly responsible for just initializing the FastAPI app. This also gives clearer control over the order of the initialization steps, which will be important as we add planned torch configurations that must be applied before torch is imported. --- invokeai/app/api_app.py | 47 ---------------------------- invokeai/app/run_app.py | 68 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index f97abd18024..b12c86b41e0 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -3,7 +3,6 @@ from contextlib import asynccontextmanager from pathlib import Path -import uvicorn from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware @@ -32,26 +31,11 @@ from invokeai.app.invocations.load_custom_nodes import load_custom_nodes from invokeai.app.services.config.config_default import get_config from invokeai.app.util.custom_openapi import get_openapi_func -from invokeai.app.util.startup_utils import ( - apply_monkeypatches, - check_cudnn, - enable_dev_reload, - find_open_port, - register_mime_types, -) -from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.logging import InvokeAILogger app_config = get_config() - -apply_monkeypatches() -register_mime_types() - logger = InvokeAILogger.get_logger(config=app_config) -torch_device_name = TorchDevice.get_torch_device_name() -logger.info(f"Using torch device: {torch_device_name}") - loop = asyncio.new_event_loop() # Load custom nodes. This must be done after importing the Graph class, which itself imports all modules from the @@ -182,34 +166,3 @@ def overridden_redoc() -> HTMLResponse: app.mount( "/static", NoCacheStaticFiles(directory=Path(web_root_path, "static/")), name="static" ) # docs favicon is in here - - -def invoke_api() -> None: - if app_config.dev_reload: - enable_dev_reload() - - orig_config_port = app_config.port - app_config.port = find_open_port(app_config.port) - if orig_config_port != app_config.port: - logger.warning(f"Port {orig_config_port} is already in use. Using port {app_config.port}.") - - check_cudnn(logger) - - config = uvicorn.Config( - app=app, - host=app_config.host, - port=app_config.port, - loop="asyncio", - log_level=app_config.log_level_network, - ssl_certfile=app_config.ssl_certfile, - ssl_keyfile=app_config.ssl_keyfile, - ) - server = uvicorn.Server(config) - - # replace uvicorn's loggers with InvokeAI's for consistent appearance - uvicorn_logger = InvokeAILogger.get_logger("uvicorn") - uvicorn_logger.handlers.clear() - for hdlr in logger.handlers: - uvicorn_logger.addHandler(hdlr) - - loop.run_until_complete(server.serve()) diff --git a/invokeai/app/run_app.py b/invokeai/app/run_app.py index 701f1dab739..cc11bd6aee7 100644 --- a/invokeai/app/run_app.py +++ b/invokeai/app/run_app.py @@ -1,12 +1,68 @@ -"""This is a wrapper around the main app entrypoint, to allow for CLI args to be parsed before running the app.""" +import uvicorn +from invokeai.app.services.config.config_default import get_config +from invokeai.app.util.startup_utils import ( + apply_monkeypatches, + check_cudnn, + enable_dev_reload, + find_open_port, + register_mime_types, +) +from invokeai.backend.util.logging import InvokeAILogger +from invokeai.frontend.cli.arg_parser import InvokeAIArgs -def run_app() -> None: - # Before doing _anything_, parse CLI args! - from invokeai.frontend.cli.arg_parser import InvokeAIArgs +def get_app(): + """Import the app and event loop. We wrap this in a function to more explicitly control when it happens, because + importing from api_app does a bunch of stuff - it's more like calling a function than importing a module. + """ + from invokeai.app.api_app import app, loop + + return app, loop + + +def run_app() -> None: + """The main entrypoint for the app.""" + # Parse the CLI arguments. InvokeAIArgs.parse_args() - from invokeai.app.api_app import invoke_api + # Load config. + app_config = get_config() + + logger = InvokeAILogger.get_logger(config=app_config) + + # Find an open port, and modify the config accordingly. + orig_config_port = app_config.port + app_config.port = find_open_port(app_config.port) + if orig_config_port != app_config.port: + logger.warning(f"Port {orig_config_port} is already in use. Using port {app_config.port}.") + + # Miscellaneous startup tasks. + apply_monkeypatches() + register_mime_types() + if app_config.dev_reload: + enable_dev_reload() + check_cudnn(logger) + + # Initialize the app and event loop. + app, loop = get_app() + + # Start the server. + config = uvicorn.Config( + app=app, + host=app_config.host, + port=app_config.port, + loop="asyncio", + log_level=app_config.log_level_network, + ssl_certfile=app_config.ssl_certfile, + ssl_keyfile=app_config.ssl_keyfile, + ) + server = uvicorn.Server(config) + + # replace uvicorn's loggers with InvokeAI's for consistent appearance + uvicorn_logger = InvokeAILogger.get_logger("uvicorn") + uvicorn_logger.handlers.clear() + for hdlr in logger.handlers: + uvicorn_logger.addHandler(hdlr) - invoke_api() + loop.run_until_complete(server.serve()) From da2b6815acf34f873793d6ba26c318dcc92d639a Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 24 Feb 2025 15:34:41 +0000 Subject: [PATCH 7/8] Make InvokeAILogger an inline import in startup_utils.py in response to review comment. --- invokeai/app/util/startup_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/invokeai/app/util/startup_utils.py b/invokeai/app/util/startup_utils.py index 9c3f194c283..726d40a7a65 100644 --- a/invokeai/app/util/startup_utils.py +++ b/invokeai/app/util/startup_utils.py @@ -4,8 +4,6 @@ import torch -from invokeai.backend.util.logging import InvokeAILogger - def find_open_port(port: int) -> int: """Find a port not in use starting at given port""" @@ -37,6 +35,8 @@ def check_cudnn(logger: logging.Logger) -> None: def enable_dev_reload() -> None: """Enable hot reloading on python file changes during development.""" + from invokeai.backend.util.logging import InvokeAILogger + try: import jurigged except ImportError as e: From 1e2c7c51b5476fce5e4420cb67833cc19cc4374c Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Fri, 28 Feb 2025 20:54:26 +0000 Subject: [PATCH 8/8] Move load_custom_nodes() to run_app() entrypoint. --- invokeai/app/api_app.py | 6 ------ invokeai/app/run_app.py | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/invokeai/app/api_app.py b/invokeai/app/api_app.py index b12c86b41e0..fda232496e7 100644 --- a/invokeai/app/api_app.py +++ b/invokeai/app/api_app.py @@ -28,7 +28,6 @@ workflows, ) from invokeai.app.api.sockets import SocketIO -from invokeai.app.invocations.load_custom_nodes import load_custom_nodes from invokeai.app.services.config.config_default import get_config from invokeai.app.util.custom_openapi import get_openapi_func from invokeai.backend.util.logging import InvokeAILogger @@ -38,11 +37,6 @@ loop = asyncio.new_event_loop() -# Load custom nodes. This must be done after importing the Graph class, which itself imports all modules from the -# invocations module. The ordering here is implicit, but important - we want to load custom nodes after all the -# core nodes have been imported so that we can catch when a custom node clobbers a core node. -load_custom_nodes(custom_nodes_path=app_config.custom_nodes_path) - @asynccontextmanager async def lifespan(app: FastAPI): diff --git a/invokeai/app/run_app.py b/invokeai/app/run_app.py index cc11bd6aee7..6eb64909927 100644 --- a/invokeai/app/run_app.py +++ b/invokeai/app/run_app.py @@ -1,5 +1,6 @@ import uvicorn +from invokeai.app.invocations.load_custom_nodes import load_custom_nodes from invokeai.app.services.config.config_default import get_config from invokeai.app.util.startup_utils import ( apply_monkeypatches, @@ -47,6 +48,11 @@ def run_app() -> None: # Initialize the app and event loop. app, loop = get_app() + # Load custom nodes. This must be done after importing the Graph class, which itself imports all modules from the + # invocations module. The ordering here is implicit, but important - we want to load custom nodes after all the + # core nodes have been imported so that we can catch when a custom node clobbers a core node. + load_custom_nodes(custom_nodes_path=app_config.custom_nodes_path) + # Start the server. config = uvicorn.Config( app=app,