diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 479913d3..62f5a542 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -3,8 +3,8 @@ { "cell_type": "markdown", "metadata": { - "id": "view-in-github", - "colab_type": "text" + "colab_type": "text", + "id": "view-in-github" }, "source": [ "\"Open" @@ -16,7 +16,7 @@ "id": "TitleTop" }, "source": [ - "# Disco Diffusion v5.6 - Now with portrait_generator_v001\n", + "# Disco Diffusion (v5.x) - Now with liminal_diffusion_v1 and liminal_diffusion_source by BrainArtLabs\n", "\n", "Disco Diffusion - http://discodiffusion.com/ , https://github.com/alembics/disco-diffusion\n", "\n", @@ -31,7 +31,7 @@ "id": "CreditsChTop" }, "source": [ - "### Credits & Changelog \u2b07\ufe0f" + "### Credits & Changelog ⬇️" ] }, { @@ -76,7 +76,9 @@ "\n", "Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11)\n", "\n", - "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)" + "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", + "\n", + "Integrated liminal_diffusion_v1 and liminal_diffusion_source by BrainArtLabs (https://twitter.com/BrainArtLabs)" ] }, { @@ -197,10 +199,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "Changelog" }, + "outputs": [], "source": [ "#@title <- View Changelog\n", "skip_for_run_all = True #@param {type: 'boolean'}\n", @@ -327,9 +331,7 @@ " portrait_generator_v001 diffusion model integrated\n", " '''\n", " )" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -415,10 +417,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "CheckGPU" }, + "outputs": [], "source": [ "#@title 1.1 Check GPU Status\n", "import subprocess\n", @@ -433,16 +437,16 @@ " print(nvidiasmi_output)\n", " nvidiasmi_ecc_note = subprocess.run(['nvidia-smi', '-i', '0', '-e', '0'], stdout=subprocess.PIPE).stdout.decode('utf-8')\n", " print(nvidiasmi_ecc_note)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "PrepFolders" }, + "outputs": [], "source": [ "#@title 1.2 Prepare Folders\n", "import subprocess, os, sys, ipykernel\n", @@ -511,16 +515,16 @@ "\n", "# libraries = f'{root_path}/libraries'\n", "# createPath(libraries)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "InstallDeps" }, + "outputs": [], "source": [ "#@title ### 1.3 Install, import dependencies and set up runtime devices\n", "\n", @@ -685,16 +689,16 @@ " if torch.cuda.get_device_capability(DEVICE) == (8,0): ## A100 fix thanks to Emad\n", " print('Disabling CUDNN for A100 gpu', file=sys.stderr)\n", " torch.backends.cudnn.enabled = False" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "DefMidasFns" }, + "outputs": [], "source": [ "#@title ### 1.4 Define Midas functions\n", "\n", @@ -797,16 +801,16 @@ "\n", " print(f\"MiDaS '{midas_model_type}' depth model initialized.\")\n", " return midas_model, midas_transform, net_w, net_h, resize_mode, normalization" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "DefFns" }, + "outputs": [], "source": [ "#@title 1.5 Define necessary functions\n", "\n", @@ -1399,12 +1403,21 @@ " except:\n", " input_resolution=224\n", "\n", - " cuts = MakeCutoutsDango(input_resolution,\n", - " Overview= args.cut_overview[1000-t_int], \n", - " InnerCrop = args.cut_innercut[1000-t_int],\n", - " IC_Size_Pow=args.cut_ic_pow[1000-t_int],\n", - " IC_Grey_P = args.cut_icgray_p[1000-t_int]\n", - " )\n", + " # liminal specific condition\n", + " if diffusion_model in liminal_model_names: \n", + " cuts = MakeCutoutsDango(input_resolution,\n", + " Overview= args.cut_overview[1000-t_int], \n", + " InnerCrop = args.cut_innercut[1000-t_int],\n", + " IC_Size_Pow = args.cut_ic_pow,\n", + " IC_Grey_P = args.cut_icgray_p[1000-t_int]\n", + " )\n", + " else: #default cuts\n", + " cuts = MakeCutoutsDango(input_resolution,\n", + " Overview = args.cut_overview[1000-t_int], \n", + " InnerCrop = args.cut_innercut[1000-t_int],\n", + " IC_Size_Pow = args.cut_ic_pow[1000-t_int],\n", + " IC_Grey_P = args.cut_icgray_p[1000-t_int]\n", + " )\n", " clip_in = normalize(cuts(x_in.add(1).div(2)))\n", " image_embeds = model_stat[\"clip_model\"].encode_image(clip_in).float()\n", " dists = spherical_dist_loss(image_embeds.unsqueeze(1), model_stat[\"target_embeds\"].unsqueeze(0))\n", @@ -1681,16 +1694,16 @@ " # print('Settings:', setting_list)\n", " with open(f\"{batchFolder}/{batch_name}({batchNum})_settings.txt\", \"w+\") as f: #save settings\n", " json.dump(setting_list, f, ensure_ascii=False, indent=4)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "DefSecModel" }, + "outputs": [], "source": [ "#@title 1.6 Define the secondary diffusion model\n", "\n", @@ -1855,9 +1868,7 @@ " pred = input * alphas - v * sigmas\n", " eps = input * sigmas + v * alphas\n", " return DiffusionOutput(v, pred, eps)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1870,12 +1881,14 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "ModelSettings" }, + "outputs": [], "source": [ "#@markdown ####**Models Settings (note: For pixel art, the best is pixelartdiffusion_expanded):**\n", - "diffusion_model = \"512x512_diffusion_uncond_finetune_008100\" #@param [\"256x256_diffusion_uncond\", \"512x512_diffusion_uncond_finetune_008100\", \"portrait_generator_v001\", \"pixelartdiffusion_expanded\", \"pixel_art_diffusion_hard_256\", \"pixel_art_diffusion_soft_256\", \"pixelartdiffusion4k\", \"watercolordiffusion_2\", \"watercolordiffusion\", \"PulpSciFiDiffusion\", \"custom\"]\n", + "diffusion_model = \"512x512_diffusion_uncond_finetune_008100\" #@param [\"256x256_diffusion_uncond\", \"512x512_diffusion_uncond_finetune_008100\", \"portrait_generator_v001\", \"pixelartdiffusion_expanded\", \"pixel_art_diffusion_hard_256\", \"pixel_art_diffusion_soft_256\", \"pixelartdiffusion4k\", \"watercolordiffusion_2\", \"watercolordiffusion\", \"PulpSciFiDiffusion\", \"liminal_diffusion_v1\", \"liminal_diffusion_source\", \"custom\"]\n", "\n", "use_secondary_model = True #@param {type: 'boolean'}\n", "diffusion_sampling_mode = 'ddim' #@param ['plms','ddim']\n", @@ -1923,13 +1936,16 @@ " 'watercolordiffusion_2': { 'downloaded': False, 'sha': '49c281b6092c61c49b0f1f8da93af9b94be7e0c20c71e662e2aa26fee0e4b1a9', 'uri_list': ['https://huggingface.co/KaliYuga/watercolordiffusion_2/resolve/main/watercolordiffusion_2.pt'] },\n", " 'watercolordiffusion': { 'downloaded': False, 'sha': 'a3e6522f0c8f278f90788298d66383b11ac763dd5e0d62f8252c962c23950bd6', 'uri_list': ['https://huggingface.co/KaliYuga/watercolordiffusion/resolve/main/watercolordiffusion.pt'] },\n", " 'PulpSciFiDiffusion': { 'downloaded': False, 'sha': 'b79e62613b9f50b8a3173e5f61f0320c7dbb16efad42a92ec94d014f6e17337f', 'uri_list': ['https://huggingface.co/KaliYuga/PulpSciFiDiffusion/resolve/main/PulpSciFiDiffusion.pt'] },\n", + " 'liminal_diffusion_v1': { 'downloaded': False, 'sha': '87c36b544a367fceb0ca127d0028cd8a6f6b6e069e529b22999259d69c14f042', 'uri_list': ['https://huggingface.co/BrainArtLabs/liminal_diffusion/resolve/main/liminal_diffusion_v1.pt'] },\n", + " 'liminal_diffusion_source': { 'downloaded': False, 'sha': 'ce0064b8cea56c8adb4e5aa0ee2d02f65cd8f1baa905cc6504f462f1aac6d6f4', 'uri_list': ['https://huggingface.co/BrainArtLabs/liminal_diffusion/resolve/main/liminal_diffusion_source.pt'] },\n", " 'secondary': { 'downloaded': False, 'sha': '983e3de6f95c88c81b2ca7ebb2c217933be1973b1ff058776b970f901584613a', 'uri_list': ['https://the-eye.eu/public/AI/models/v-diffusion/secondary_model_imagenet_2.pth', 'https://ipfs.pollinations.ai/ipfs/bafybeibaawhhk7fhyhvmm7x24zwwkeuocuizbqbcg5nqx64jq42j75rdiy/secondary_model_imagenet_2.pth'] },\n", "}\n", "\n", "kaliyuga_pixel_art_model_names = ['pixelartdiffusion_expanded', 'pixel_art_diffusion_hard_256', 'pixel_art_diffusion_soft_256', 'pixelartdiffusion4k', 'PulpSciFiDiffusion']\n", "kaliyuga_watercolor_model_names = ['watercolordiffusion', 'watercolordiffusion_2']\n", "kaliyuga_pulpscifi_model_names = ['PulpSciFiDiffusion']\n", - "diffusion_models_256x256_list = ['256x256_diffusion_uncond'] + kaliyuga_pixel_art_model_names + kaliyuga_watercolor_model_names + kaliyuga_pulpscifi_model_names\n", + "liminal_model_names = ['liminal_diffusion_v1', 'liminal_diffusion_source'] #added\n", + "diffusion_models_256x256_list = ['256x256_diffusion_uncond'] + kaliyuga_pixel_art_model_names + kaliyuga_watercolor_model_names + kaliyuga_pulpscifi_model_names + liminal_model_names\n", "\n", "from urllib.parse import urlparse\n", "\n", @@ -2027,6 +2043,23 @@ " 'use_fp16': True,\n", " 'use_scale_shift_norm': True,\n", " })\n", + "elif diffusion_model == 'liminal_diffusion_v1' or diffusion_model == 'liminal_diffusion_source':\n", + " model_config.update({\n", + " 'attention_resolutions': '16',\n", + " 'class_cond': False,\n", + " 'diffusion_steps': 1000,\n", + " 'rescale_timesteps': True,\n", + " 'timestep_respacing': 'ddim100',\n", + " 'image_size': 256,\n", + " 'learn_sigma': True,\n", + " 'noise_schedule': 'linear',\n", + " 'num_channels': 128,\n", + " 'num_heads': 1,\n", + " 'num_res_blocks': 2,\n", + " 'use_checkpoint': use_checkpoint,\n", + " 'use_fp16': True,\n", + " 'use_scale_shift_norm': False,\n", + " })\n", "else: # E.g. A model finetuned by KaliYuga\n", " model_config.update({\n", " 'attention_resolutions': '16',\n", @@ -2078,9 +2111,7 @@ "\n", "normalize = T.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711])\n", "lpips_model = lpips.LPIPS(net='vgg').to(device)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -2094,9 +2125,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "CustModel" }, + "outputs": [], "source": [ "#@markdown ####**Custom Model Settings:**\n", "if diffusion_model == 'custom':\n", @@ -2116,9 +2149,7 @@ " 'use_fp16': True,\n", " 'use_scale_shift_norm': False,\n", " })" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -2131,9 +2162,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "BasicSettings" }, + "outputs": [], "source": [ "#@markdown ####**Basic Settings:**\n", "batch_name = 'TimeToDisco' #@param{type: 'string'}\n", @@ -2177,9 +2210,7 @@ "#Make folder for batch\n", "batchFolder = f'{outDirPath}/{batch_name}'\n", "createPath(batchFolder)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -2192,9 +2223,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "AnimSettings" }, + "outputs": [], "source": [ "#@markdown ####**Animation Mode:**\n", "animation_mode = 'None' #@param ['None', '2D', '3D', 'Video Input'] {type:'string'}\n", @@ -2564,15 +2597,15 @@ " rotation_3d_x = float(rotation_3d_x)\n", " rotation_3d_y = float(rotation_3d_y)\n", " rotation_3d_z = float(rotation_3d_z)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "InstallRAFT" }, + "outputs": [], "source": [ "#@title Install RAFT for Video input animation mode only\n", "#@markdown Run once per session. Doesn't download again if model path exists.\n", @@ -2592,15 +2625,15 @@ " sub_p_res = subprocess.run(['bash', f'{PROJECT_DIR}/RAFT/download_models.sh'], stdout=subprocess.PIPE).stdout.decode('utf-8')\n", " print(sub_p_res)\n", " os.chdir(PROJECT_DIR)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "FlowFns1" }, + "outputs": [], "source": [ "#@title Define optical flow functions for Video input animation mode only\n", "if animation_mode == 'Video Input':\n", @@ -2738,15 +2771,15 @@ " # TBD flow backwards!\n", " \n", " os.chdir(PROJECT_DIR)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "FlowFns2" }, + "outputs": [], "source": [ "#@title Generate optical flow and consistency maps\n", "#@markdown Run once per init video\n", @@ -2804,9 +2837,7 @@ "\n", " del raft_model \n", " gc.collect()" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -2820,9 +2851,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "ExtraSettings" }, + "outputs": [], "source": [ "#@markdown ####**Saving:**\n", "\n", @@ -2894,6 +2927,12 @@ "watercolor_cut_ic_pow = \"[12]*300+[12]*100+[12]*50+[12]*50+[10]*100+[10]*100+[10]*300\" #@param {type: 'string'}\n", "watercolor_cut_icgray_p = \"[0.7]*100+[0.6]*100+[0.45]*100+[0.3]*100+[0]*600\" #@param {type: 'string'}\n", "\n", + "#@markdown **Used for liminal diffusion models**\n", + "lim_cut_overview = \"[16]*400+[4]*600\" #@param {type: 'string'} #added\n", + "lim_cut_innercut =\"[4]*400+[16]*600\" #@param {type: 'string'} #added\n", + "lim_cut_ic_pow = \"2\" #@param {type: 'string'} #added\n", + "lim_cut_icgray_p = \"[0.2]*400+[0.1]*600\" #@param {type: 'string'} #added\n", + "\n", "if (diffusion_model in kaliyuga_pixel_art_model_names) or (diffusion_model in kaliyuga_pulpscifi_model_names):\n", " cut_overview = pad_or_pulp_cut_overview\n", " cut_innercut = pad_or_pulp_cut_innercut\n", @@ -2904,6 +2943,11 @@ " cut_innercut = watercolor_cut_innercut\n", " cut_ic_pow = watercolor_cut_ic_pow\n", " cut_icgray_p = watercolor_cut_icgray_p\n", + "elif diffusion_model in liminal_model_names: #added\n", + " cut_overview = lim_cut_overview\n", + " cut_innercut = lim_cut_innercut\n", + " cut_ic_pow = lim_cut_ic_pow\n", + " cut_icgray_p = lim_cut_icgray_p\n", "\n", "#@markdown ---\n", "\n", @@ -2911,9 +2955,7 @@ "use_vertical_symmetry = False #@param {type:\"boolean\"}\n", "use_horizontal_symmetry = False #@param {type:\"boolean\"}\n", "transformation_percent = [0.09] #@param" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -2927,9 +2969,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "Prompts" }, + "outputs": [], "source": [ "# Note: If using a pixelart diffusion model, try adding \"#pixelart\" to the end of the prompt for a stronger effect. It'll tend to work a lot better!\n", "text_prompts = {\n", @@ -2940,9 +2984,7 @@ "image_prompts = {\n", " # 0:['ImagePromptsWorkButArentVeryGood.png:2',],\n", "}" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -2955,9 +2997,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "DoTheRun" }, + "outputs": [], "source": [ "#@title Do the Run!\n", "#@markdown `n_batches` ignored with animation modes.\n", @@ -3183,9 +3227,7 @@ " print('Seed used:', seed)\n", " gc.collect()\n", " torch.cuda.empty_cache()" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -3198,10 +3240,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "CreateVid", - "cellView": "form" + "cellView": "form", + "id": "CreateVid" }, + "outputs": [], "source": [ "import PIL\n", "# @title ### **Create video**\n", @@ -3328,14 +3372,12 @@ " # mp4 = open(filepath,'rb').read()\n", " # data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n", " # display.HTML(f'')" - ], - "outputs": [], - "execution_count": null + ] } ], "metadata": { - "anaconda-cloud": {}, "accelerator": "GPU", + "anaconda-cloud": {}, "colab": { "collapsed_sections": [ "CreditsChTop", @@ -3353,14 +3395,14 @@ "FlowFns1", "FlowFns2" ], + "include_colab_link": true, "machine_shape": "hm", "name": "Disco Diffusion v5.6 [Now with portrait_generator_v001]", "private_outputs": true, - "provenance": [], - "include_colab_link": true + "provenance": [] }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10.5 64-bit", "language": "python", "name": "python3" }, @@ -3374,9 +3416,14 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.1" + "version": "3.10.5" + }, + "vscode": { + "interpreter": { + "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" + } } }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +}