From fe6765e763ebf3c69da54658a10a6e48caf4fea0 Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:32:02 +0200 Subject: [PATCH 1/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 48 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 7518f6fa..2ef175ac 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -16,7 +16,7 @@ "id": "TitleTop" }, "source": [ - "# Disco Diffusion v5.7 - Now with MiDaS (3D mode) not being broken\n", + "# Disco Diffusion v5.8 - Now with stsp and ltsp sampling\n", "\n", "Disco Diffusion - http://discodiffusion.com/ , https://github.com/alembics/disco-diffusion\n", "\n", @@ -76,7 +76,9 @@ "\n", "Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11)\n", "\n", - "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)" + "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", + "\n", + "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)\n", ] }, { @@ -334,6 +336,8 @@ " v5.7 Update: Dec 31st 2022 - Steffen Moelter (with minor colab-convert integration by gandamu)\n", "\n", " Clone MiDaS v3 specifically. This fixes 3D mode. It had been broken since MiDaS v3.1 introduced an incompatibility.\n", + "\n", + " v5.8 Update: Jun 5st 2023 - Added STSP and LTSP Sampling with Splitting Numerical Methods from space-nuko repository. (integration by DrSen)\n", " '''\n", " )" ], @@ -596,7 +600,7 @@ " from guided_diffusion.script_util import create_model_and_diffusion\n", "except:\n", " if not os.path.exists(\"guided-diffusion\"):\n", - " gitclone(\"https://github.com/kostarion/guided-diffusion\")\n", + " gitclone(\"https://github.com/space-nuko/guided-diffusion/tree/f91a8708b7b05b48ad01b56bbf5ba375fc66818a\")\n", " sys.path.append(f'{PROJECT_DIR}/guided-diffusion')\n", "\n", "try:\n", @@ -1453,6 +1457,10 @@ " \n", " if args.diffusion_sampling_mode == 'ddim':\n", " sample_fn = diffusion.ddim_sample_loop_progressive\n", + " elif args.diffusion_sampling_mode == 'ltsp': + " sample_fn = diffusion.ltsp_sample_loop_progressive + " elif args.diffusion_sampling_mode == 'stsp': + " sample_fn = diffusion.stsp_sample_loop_progressive " else:\n", " sample_fn = diffusion.plms_sample_loop_progressive\n", "\n", @@ -1489,6 +1497,34 @@ " transformation_fn=symmetry_transformation_fn,\n", " transformation_percent=args.transformation_percent\n", " )\n", + " if args.diffusion_sampling_mode == 'stsp':\n", + " samples = sample_fn(\n", + " model,\n", + " (batch_size, 3, args.side_y, args.side_x),\n", + " clip_denoised=clip_denoised,\n", + " model_kwargs={},\n", + " cond_fn=cond_fn,\n", + " impu_fn=None,\n", + " progress=True,\n", + " skip_timesteps=skip_steps,\n", + " init_image=init,\n", + " randomize_class=randomize_class,\n", + " order=2,\n", + " )\n", + " if args.diffusion_sampling_mode == 'ltsp':\n", + " samples = sample_fn(\n", + " model,\n", + " (batch_size, 3, args.side_y, args.side_x),\n", + " clip_denoised=clip_denoised,\n", + " model_kwargs={},\n", + " cond_fn=cond_fn,\n", + " impu_fn=None,\n", + " progress=True,\n", + " skip_timesteps=skip_steps,\n", + " init_image=init,\n", + " randomize_class=randomize_class,\n", + " order=2,\n", + " )\n", " else:\n", " samples = sample_fn(\n", " model,\n", @@ -1895,7 +1931,7 @@ "diffusion_model = \"512x512_diffusion_uncond_finetune_008100\" #@param [\"256x256_diffusion_uncond\", \"512x512_diffusion_uncond_finetune_008100\", \"portrait_generator_v001\", \"pixelartdiffusion_expanded\", \"pixel_art_diffusion_hard_256\", \"pixel_art_diffusion_soft_256\", \"pixelartdiffusion4k\", \"watercolordiffusion_2\", \"watercolordiffusion\", \"PulpSciFiDiffusion\", \"custom\"]\n", "\n", "use_secondary_model = True #@param {type: 'boolean'}\n", - "diffusion_sampling_mode = 'ddim' #@param ['plms','ddim']\n", + "diffusion_sampling_mode = 'stsp' #@param ['plms','ddim','stsp','ltsp']\n", "#@markdown #####**Custom model:**\n", "custom_path = '/content/drive/MyDrive/deep_learning/ddpm/ema_0.9999_058000.pt'#@param {type: 'string'}\n", "\n", @@ -2154,7 +2190,7 @@ "source": [ "#@markdown ####**Basic Settings:**\n", "batch_name = 'TimeToDisco' #@param{type: 'string'}\n", - "steps = 250 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true}\n", + "steps = 150 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true}\n", "width_height_for_512x512_models = [1280, 768] #@param{type: 'raw'}\n", "clip_guidance_scale = 5000 #@param{type: 'number'}\n", "tv_scale = 0#@param{type: 'number'}\n", @@ -3371,7 +3407,7 @@ "FlowFns2" ], "machine_shape": "hm", - "name": "Disco Diffusion v5.7 [Now with MiDaS (3D mode) not being broken]", + "name": "Disco Diffusion v5.8 - Now with stsp and ltsp sampling", "private_outputs": true, "provenance": [], "include_colab_link": true From d1e70b36fbd60ad388ec00887891b52437be1da6 Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:40:46 +0200 Subject: [PATCH 2/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 2ef175ac..2b570761 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -1457,10 +1457,10 @@ " \n", " if args.diffusion_sampling_mode == 'ddim':\n", " sample_fn = diffusion.ddim_sample_loop_progressive\n", - " elif args.diffusion_sampling_mode == 'ltsp': - " sample_fn = diffusion.ltsp_sample_loop_progressive - " elif args.diffusion_sampling_mode == 'stsp': - " sample_fn = diffusion.stsp_sample_loop_progressive + " elif args.diffusion_sampling_mode == 'ltsp':\n", + " sample_fn = diffusion.ltsp_sample_loop_progressive\n", + " elif args.diffusion_sampling_mode == 'stsp':\n", + " sample_fn = diffusion.stsp_sample_loop_progressive\n", " else:\n", " sample_fn = diffusion.plms_sample_loop_progressive\n", "\n", From 3bc80e0d73358f0b6d340097337793b9de40d04d Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:45:02 +0200 Subject: [PATCH 3/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 2b570761..739ff053 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -78,7 +78,7 @@ "\n", "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", "\n", - "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)\n", + "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)", ] }, { From 17b7a2dcc09b3e552ee3c2afa672e66fe8e4c7c3 Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:52:11 +0200 Subject: [PATCH 4/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 739ff053..ab6cb9f9 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -600,7 +600,7 @@ " from guided_diffusion.script_util import create_model_and_diffusion\n", "except:\n", " if not os.path.exists(\"guided-diffusion\"):\n", - " gitclone(\"https://github.com/space-nuko/guided-diffusion/tree/f91a8708b7b05b48ad01b56bbf5ba375fc66818a\")\n", + " gitclone(\"https://github.com/space-nuko/guided-diffusion.git\")\n", " sys.path.append(f'{PROJECT_DIR}/guided-diffusion')\n", "\n", "try:\n", From bf7a776b696c9f7e28d548d9ed5d14ee8c5158d9 Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:53:40 +0200 Subject: [PATCH 5/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index ab6cb9f9..a7bd40d2 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -76,9 +76,9 @@ "\n", "Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11)\n", "\n", - "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", + "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)", "\n", - "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)", + "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)\n", ] }, { From 789ba7d616dad401d320611190a700756c16e7ab Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:54:56 +0200 Subject: [PATCH 6/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index a7bd40d2..ed9e5bb0 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -76,9 +76,10 @@ "\n", "Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11)\n", "\n", - "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)", + "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", "\n", "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)\n", + "\n", ] }, { From 1d32578a8f30da3e0cc97e11d02eda8cb0cd2aec Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Mon, 5 Jun 2023 20:03:51 +0200 Subject: [PATCH 7/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index ed9e5bb0..417e0a0a 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -78,8 +78,7 @@ "\n", "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", "\n", - "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)\n", - "\n", + "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)" ] }, { From d6c48f368c5422cfe58c2387a18b1f08a20f9d65 Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Wed, 7 Jun 2023 22:51:26 +0200 Subject: [PATCH 8/9] Update disco.py --- disco.py | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/disco.py b/disco.py index 65dce7a2..392c980d 100644 --- a/disco.py +++ b/disco.py @@ -12,7 +12,7 @@ # !! "id": "TitleTop" # !! }} """ -# Disco Diffusion v5.7 - Now with MiDaS (3D mode) not being broken +# Disco Diffusion v5.8 - Now with stsp and ltsp sampling Disco Diffusion - http://discodiffusion.com/ , https://github.com/alembics/disco-diffusion @@ -71,6 +71,8 @@ Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11) Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist) + +Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion) """ # %% @@ -323,6 +325,8 @@ v5.7 Update: Dec 31st 2022 - Steffen Moelter (with minor colab-convert integration by gandamu) Clone MiDaS v3 specifically. This fixes 3D mode. It had been broken since MiDaS v3.1 introduced an incompatibility. + + v5.8 Update: Jun 5st 2023 - Added STSP and LTSP Sampling with Splitting Numerical Methods from space-nuko repository. (integration by DrSen) ''' ) @@ -567,7 +571,7 @@ def createPath(filepath): from guided_diffusion.script_util import create_model_and_diffusion except: if not os.path.exists("guided-diffusion"): - gitclone("https://github.com/kostarion/guided-diffusion") + gitclone("https://github.com/space-nuko/guided-diffusion.git") sys.path.append(f'{PROJECT_DIR}/guided-diffusion') try: @@ -1414,6 +1418,10 @@ def cond_fn(x, t, y=None): if args.diffusion_sampling_mode == 'ddim': sample_fn = diffusion.ddim_sample_loop_progressive + elif args.diffusion_sampling_mode == 'ltsp': + sample_fn = diffusion.ltsp_sample_loop_progressive + elif args.diffusion_sampling_mode == 'stsp': + sample_fn = diffusion.stsp_sample_loop_progressive else: sample_fn = diffusion.plms_sample_loop_progressive @@ -1450,6 +1458,34 @@ def cond_fn(x, t, y=None): transformation_fn=symmetry_transformation_fn, transformation_percent=args.transformation_percent ) + if args.diffusion_sampling_mode == 'stsp': + samples = sample_fn( + model, + (batch_size, 3, args.side_y, args.side_x), + clip_denoised=clip_denoised, + model_kwargs={}, + cond_fn=cond_fn, + impu_fn=None, + progress=True, + skip_timesteps=skip_steps, + init_image=init, + randomize_class=randomize_class, + order=2, + ) + if args.diffusion_sampling_mode == 'ltsp': + samples = sample_fn( + model, + (batch_size, 3, args.side_y, args.side_x), + clip_denoised=clip_denoised, + model_kwargs={}, + cond_fn=cond_fn, + impu_fn=None, + progress=True, + skip_timesteps=skip_steps, + init_image=init, + randomize_class=randomize_class, + order=2, + ) else: samples = sample_fn( model, @@ -1845,7 +1881,7 @@ def forward(self, input, t): diffusion_model = "512x512_diffusion_uncond_finetune_008100" #@param ["256x256_diffusion_uncond", "512x512_diffusion_uncond_finetune_008100", "portrait_generator_v001", "pixelartdiffusion_expanded", "pixel_art_diffusion_hard_256", "pixel_art_diffusion_soft_256", "pixelartdiffusion4k", "watercolordiffusion_2", "watercolordiffusion", "PulpSciFiDiffusion", "custom"] use_secondary_model = True #@param {type: 'boolean'} -diffusion_sampling_mode = 'ddim' #@param ['plms','ddim'] +diffusion_sampling_mode = 'stsp' #@param ['plms','ddim','stsp','ltsp'] #@markdown #####**Custom model:** custom_path = '/content/drive/MyDrive/deep_learning/ddpm/ema_0.9999_058000.pt'#@param {type: 'string'} @@ -2092,7 +2128,7 @@ def download_model(diffusion_model_name, uri_index=0): # !! }} #@markdown ####**Basic Settings:** batch_name = 'TimeToDisco' #@param{type: 'string'} -steps = 250 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true} +steps = 142 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true} width_height_for_512x512_models = [1280, 768] #@param{type: 'raw'} clip_guidance_scale = 5000 #@param{type: 'number'} tv_scale = 0#@param{type: 'number'} @@ -3264,7 +3300,7 @@ def move_files(start_num, end_num, old_folder, new_folder): # !! "FlowFns2" # !! ], # !! "machine_shape": "hm", -# !! "name": "Disco Diffusion v5.61 [Now with portrait_generator_v001]", +# !! "name": "Disco Diffusion v5.8 - Now with stsp and ltsp sampling", # !! "private_outputs": true, # !! "provenance": [], # !! "include_colab_link": true From 1442bb071bd432bc87ccad7c43a1e0ad0384fad7 Mon Sep 17 00:00:00 2001 From: Dr-Sen <133710952+Dr-Sen@users.noreply.github.com> Date: Wed, 7 Jun 2023 22:57:23 +0200 Subject: [PATCH 9/9] Update Disco_Diffusion.ipynb --- Disco_Diffusion.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 417e0a0a..24d75b34 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -44,7 +44,7 @@ "\n", "Original notebook by Katherine Crowson (https://github.com/crowsonkb, https://twitter.com/RiversHaveWings). It uses either OpenAI's 256x256 unconditional ImageNet or Katherine Crowson's fine-tuned 512x512 diffusion model (https://github.com/openai/guided-diffusion), together with CLIP (https://github.com/openai/CLIP) to connect text prompts with images.\n", "\n", - "Modified by Daniel Russell (https://github.com/russelldc, https://twitter.com/danielrussruss) to include (hopefully) optimal params for quick generations in 15-100 timesteps rather than 1000, as well as more robust augmentations.\n", + "Modified by Daniel Russell (https://github.com/russelldc, https://twitter.com/danielrussruss) to include (hopefully) optimal params for quick generations in 15-100 timess rather than 1000, as well as more robust augmentations.\n", "\n", "Further improvements from Dango233 and nshepperd helped improve the quality of diffusion in general, and especially so for shorter runs like this notebook aims to achieve.\n", "\n", @@ -2190,7 +2190,7 @@ "source": [ "#@markdown ####**Basic Settings:**\n", "batch_name = 'TimeToDisco' #@param{type: 'string'}\n", - "steps = 150 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true}\n", + "steps = 142 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true}\n", "width_height_for_512x512_models = [1280, 768] #@param{type: 'raw'}\n", "clip_guidance_scale = 5000 #@param{type: 'number'}\n", "tv_scale = 0#@param{type: 'number'}\n",