From fd9aa416fb3afa5fe603cb47fd70fbf777f9ede6 Mon Sep 17 00:00:00 2001 From: aztecman Date: Wed, 20 Jul 2022 14:07:29 -0700 Subject: [PATCH] Added Diagonal and Radial Symmetry Diagonal, Radial, and advanced settings for choosing the order of operations. --- Disco_Diffusion.ipynb | 162 ++++++++++++++++++++++++++++++++++++------ disco.py | 159 +++++++++++++++++++++++++++++++++++------ 2 files changed, 277 insertions(+), 44 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 479913d3..f3fafc9e 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -3,8 +3,8 @@ { "cell_type": "markdown", "metadata": { - "id": "view-in-github", - "colab_type": "text" + "colab_type": "text", + "id": "view-in-github" }, "source": [ "\"Open" @@ -325,6 +325,10 @@ " v5.6 Update: Jul 13th 2022 - Felipe3DArtist integration by gandamu / Adam Letts\n", "\n", " portrait_generator_v001 diffusion model integrated\n", + " \n", + " v5.7 Update: Jul 20th 2022 - Aztecman / Carson Bentley\n", + "\n", + " Diagonal Symmetry, Radial Symmetry\n", " '''\n", " )" ], @@ -1128,16 +1132,110 @@ " sampling_mode=args.sampling_mode, midas_weight=args.midas_weight)\n", " return next_step_pil\n", "\n", + "def square_pad(image):\n", + " img_size = image.size()[2:]\n", + " max_wh = max(img_size)\n", + " p_top, p_left = [(max_wh - s) // 2 for s in img_size]\n", + " p_bottom, p_right = [max_wh - (s+pad) for s, pad in zip(img_size, [p_top, p_left])]\n", + " padding = (p_left, p_top, p_right, p_bottom)\n", + " return TF.pad(image, padding, 0, 'constant'), padding\n", + "\n", + "def remove_pad(image, pad):\n", + " h, w = image.size()[2:]\n", + " return image[:,:,pad[3]:h-pad[1],pad[0]:w-pad[2]]\n", + "\n", + "def triu_secondary(x, diag_offset):\n", + " mask = torch.ones_like(x)\n", + " mask = torch.triu(mask, diagonal=diag_offset)\n", + " mask = TF.hflip(mask)\n", + " return x * mask\n", + "\n", + "class SymmTransforms:\n", + " \"\"\"\n", + " Symmetry Transforms\n", + " \"\"\"\n", + " def __init__(self, x_shape):\n", + " [self.n, self.c, self.h, self.w] = x_shape\n", + " pass\n", + "\n", + " def horizontal(self, x):\n", + " print(\"horizontal symmetry applied\")\n", + " return torch.concat((x[:, :, :, :self.w//2], torch.flip(x[:, :, :, :self.w//2], [-1])), -1)\n", + " \n", + " def vertical(self, x):\n", + " print(\"vertical symmetry applied\")\n", + " return torch.concat((x[:, :, :self.h//2, :], torch.flip(x[:, :, :self.h//2, :], [-2])), -2)\n", + " \n", + " def diagonal_pos(self, x):\n", + " if self.h != self.w:\n", + " raise ValueError(\"height must equal width for diagonal symmetry\")\n", + " print(\"diagonal symmetry mirroring across the line of y = x\")\n", + " x_triu = triu_secondary(x, 1)\n", + " x_triu_flip = TF.hflip(torch.rot90(triu_secondary(x, 0), 1, (2, 3)))\n", + " return x_triu + x_triu_flip\n", + " \n", + " def diagonal_neg(self, x):\n", + " if self.h != self.w:\n", + " raise ValueError(\"height must equal width for diagonal symmetry\")\n", + " print(\"diagonal symmetry mirroring across the line of y = -x\")\n", + " x_triu = torch.triu(x, 1)\n", + " x_triu_flip = TF.vflip(torch.rot90(torch.triu(x, 0), 1, (2, 3)))\n", + " return x_triu + x_triu_flip\n", + "\n", + " def radial(self, x, num_rays):\n", + " [n, c, h, w] = [self.n, self.c, self.h, self.w]\n", + " pad = (0,0,0,0)\n", + " if self.h != self.w:\n", + " x, pad = square_pad(x)\n", + " [n, c, h, w] = x.size() \n", + " mask = torch.triu(torch.ones_like(x))\n", + " if self.w <= self.h: #default: main slice at 12 o-clock pointing down\n", + " mask = TF.rotate(mask, (360/(num_rays*2)) - 45)\n", + " masked = mask * x\n", + " pizza_slice = torch.concat((masked[:, :, :, :w//2], TF.hflip(masked[:, :, :, :w//2])), -1)\n", + " else: #if landscape format, main slice at 9 o-clock pointing right\n", + " mask = TF.rotate(mask, (360/(num_rays*2)) + 45)\n", + " masked = mask * x\n", + " pizza_slice = torch.concat((TF.vflip(masked[:, :, h//2:, :]), masked[:, :, h//2:, :]), -2) \n", + " pizza = torch.zeros_like(x)\n", + " for i in range(num_rays):\n", + " pizza += TF.rotate(pizza_slice, i*(360/num_rays)) \n", + " if self.h != self.w:\n", + " pizza = remove_pad(pizza, pad) \n", + " return pizza\n", + "\n", "def symmetry_transformation_fn(x):\n", + " [n, c, h, w] = x.size()\n", + " if len(args.override_str) > 0 and args.override_str != \".\":\n", + " #using override string\n", + " func_dict = {\"/\":\"diagonal_pos\",\n", + " \"\\\\\":\"diagonal_neg\",\n", + " \"-\":\"vertical\",\n", + " \"|\":\"horizontal\",\n", + " \"*\":\"radial\"}\n", + " m = globals()['SymmTransforms'](x.size())\n", + " for char in args.override_str:\n", + " if char == \".\":\n", + " continue\n", + " t_func = getattr(m, func_dict[char])\n", + " if char == \"*\":\n", + " x = t_func(x, args.n_rays)\n", + " else:\n", + " x = t_func(x)\n", + " else:\n", + " #if no override, using boolean parameters\n", + " symm_t = SymmTransforms(x.size())\n", + " if args.use_diagonal_pos_symmetry:\n", + " x = symm_t.diagonal_pos(x)\n", + " if args.use_diagonal_neg_symmetry:\n", + " x = symm_t.diagonal_neg(x)\n", " if args.use_horizontal_symmetry:\n", - " [n, c, h, w] = x.size()\n", - " x = torch.concat((x[:, :, :, :w//2], torch.flip(x[:, :, :, :w//2], [-1])), -1)\n", - " print(\"horizontal symmetry applied\")\n", + " x = symm_t.horizontal(x)\n", " if args.use_vertical_symmetry:\n", - " [n, c, h, w] = x.size()\n", - " x = torch.concat((x[:, :, :h//2, :], torch.flip(x[:, :, :h//2, :], [-2])), -2)\n", - " print(\"vertical symmetry applied\")\n", - " return x\n", + " x = symm_t.vertical(x)\n", + " if args.use_radial_symmetry:\n", + " x = symm_t.radial(x, args.n_rays)\n", + " return x\n", "\n", "def do_run():\n", " seed = args.seed\n", @@ -1659,8 +1757,13 @@ " 'turbo_mode':turbo_mode,\n", " 'turbo_steps':turbo_steps,\n", " 'turbo_preroll':turbo_preroll,\n", - " 'use_horizontal_symmetry':use_horizontal_symmetry,\n", - " 'use_vertical_symmetry':use_vertical_symmetry,\n", + " 'use_horizontal_symmetry': use_horizontal_symmetry,\n", + " 'use_vertical_symmetry': use_vertical_symmetry,\n", + " 'use_diagonal_pos_symmetry': use_diagonal_pos_symmetry,\n", + " 'use_diagonal_neg_symmetry': use_diagonal_neg_symmetry,\n", + " 'use_radial_symmetry': use_radial_symmetry,\n", + " 'n_rays': n_rays,\n", + " 'override_str':override_str,\n", " 'transformation_percent':transformation_percent,\n", " #video init settings\n", " 'video_init_steps': video_init_steps,\n", @@ -1679,7 +1782,7 @@ " 'video_init_blend_mode':video_init_blend_mode\n", " }\n", " # print('Settings:', setting_list)\n", - " with open(f\"{batchFolder}/{batch_name}({batchNum})_settings.txt\", \"w+\") as f: #save settings\n", + " with open(f\"{batchFolder}/{batch_name}({batchNum})_settings.txt\", \"w+\", encoding=\"utf-8\") as f: #save settings\n", " json.dump(setting_list, f, ensure_ascii=False, indent=4)" ], "outputs": [], @@ -2906,11 +3009,21 @@ " cut_icgray_p = watercolor_cut_icgray_p\n", "\n", "#@markdown ---\n", + "#@markdown diagonal symmetry requires a square image. n_rays must be at least 1 (for radial symmetry).\n", "\n", "#@markdown ####**Transformation Settings:**\n", - "use_vertical_symmetry = False #@param {type:\"boolean\"}\n", + "use_diagonal_pos_symmetry = False #@param {type:\"boolean\"}\n", + "use_diagonal_neg_symmetry = False #@param {type:\"boolean\"}\n", "use_horizontal_symmetry = False #@param {type:\"boolean\"}\n", - "transformation_percent = [0.09] #@param" + "use_vertical_symmetry = False #@param {type:\"boolean\"}\n", + "use_radial_symmetry = False #@param {type:\"boolean\"}\n", + "n_rays = 1 #@param{type: 'number'}\n", + "transformation_percent = [0.09] #@param\n", + "\n", + "### ADVANCED SYMMETRY:\n", + "# override_str can be used to change the order of transforms, For example r\"|-/\\*.\" means to apply horizontal (|), vertical (-), diagonal+ (/), diagonal- (\\), radial (*).\n", + "# note, override_str must end in a '.'\n", + "override_str = r\".\"" ], "outputs": [], "execution_count": null @@ -3126,8 +3239,13 @@ " 'turbo_mode':turbo_mode,\n", " 'turbo_steps':turbo_steps,\n", " 'turbo_preroll':turbo_preroll,\n", - " 'use_vertical_symmetry': use_vertical_symmetry,\n", " 'use_horizontal_symmetry': use_horizontal_symmetry,\n", + " 'use_vertical_symmetry': use_vertical_symmetry,\n", + " 'use_diagonal_pos_symmetry': use_diagonal_pos_symmetry,\n", + " 'use_diagonal_neg_symmetry': use_diagonal_neg_symmetry,\n", + " 'use_radial_symmetry': use_radial_symmetry,\n", + " 'n_rays': n_rays,\n", + " 'override_str':override_str,\n", " 'transformation_percent': transformation_percent,\n", " #video init settings\n", " 'video_init_steps': video_init_steps,\n", @@ -3199,8 +3317,8 @@ { "cell_type": "code", "metadata": { - "id": "CreateVid", - "cellView": "form" + "cellView": "form", + "id": "CreateVid" }, "source": [ "import PIL\n", @@ -3334,8 +3452,8 @@ } ], "metadata": { - "anaconda-cloud": {}, "accelerator": "GPU", + "anaconda-cloud": {}, "colab": { "collapsed_sections": [ "CreditsChTop", @@ -3353,14 +3471,14 @@ "FlowFns1", "FlowFns2" ], + "include_colab_link": true, "machine_shape": "hm", "name": "Disco Diffusion v5.6 [Now with portrait_generator_v001]", "private_outputs": true, - "provenance": [], - "include_colab_link": true + "provenance": [] }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -3374,7 +3492,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.1" + "version": "3.9.12" } }, "nbformat": 4, diff --git a/disco.py b/disco.py index 1e3111e4..7b279cca 100644 --- a/disco.py +++ b/disco.py @@ -1,7 +1,7 @@ # %% # !! {"metadata":{ -# !! "id": "view-in-github", -# !! "colab_type": "text" +# !! "colab_type": "text", +# !! "id": "view-in-github" # !! }} """ Open In Colab @@ -314,6 +314,10 @@ v5.6 Update: Jul 13th 2022 - Felipe3DArtist integration by gandamu / Adam Letts portrait_generator_v001 diffusion model integrated + + v5.7 Update: Jul 20th 2022 - Aztecman / Carson Bentley + + Diagonal Symmetry, Radial Symmetry ''' ) @@ -1089,16 +1093,110 @@ def do_3d_step(img_filepath, frame_num, midas_model, midas_transform): sampling_mode=args.sampling_mode, midas_weight=args.midas_weight) return next_step_pil +def square_pad(image): + img_size = image.size()[2:] + max_wh = max(img_size) + p_top, p_left = [(max_wh - s) // 2 for s in img_size] + p_bottom, p_right = [max_wh - (s+pad) for s, pad in zip(img_size, [p_top, p_left])] + padding = (p_left, p_top, p_right, p_bottom) + return TF.pad(image, padding, 0, 'constant'), padding + +def remove_pad(image, pad): + h, w = image.size()[2:] + return image[:,:,pad[3]:h-pad[1],pad[0]:w-pad[2]] + +def triu_secondary(x, diag_offset): + mask = torch.ones_like(x) + mask = torch.triu(mask, diagonal=diag_offset) + mask = TF.hflip(mask) + return x * mask + +class SymmTransforms: + """ + Symmetry Transforms + """ + def __init__(self, x_shape): + [self.n, self.c, self.h, self.w] = x_shape + pass + + def horizontal(self, x): + print("horizontal symmetry applied") + return torch.concat((x[:, :, :, :self.w//2], torch.flip(x[:, :, :, :self.w//2], [-1])), -1) + + def vertical(self, x): + print("vertical symmetry applied") + return torch.concat((x[:, :, :self.h//2, :], torch.flip(x[:, :, :self.h//2, :], [-2])), -2) + + def diagonal_pos(self, x): + if self.h != self.w: + raise ValueError("height must equal width for diagonal symmetry") + print("diagonal symmetry mirroring across the line of y = x") + x_triu = triu_secondary(x, 1) + x_triu_flip = TF.hflip(torch.rot90(triu_secondary(x, 0), 1, (2, 3))) + return x_triu + x_triu_flip + + def diagonal_neg(self, x): + if self.h != self.w: + raise ValueError("height must equal width for diagonal symmetry") + print("diagonal symmetry mirroring across the line of y = -x") + x_triu = torch.triu(x, 1) + x_triu_flip = TF.vflip(torch.rot90(torch.triu(x, 0), 1, (2, 3))) + return x_triu + x_triu_flip + + def radial(self, x, num_rays): + [n, c, h, w] = [self.n, self.c, self.h, self.w] + pad = (0,0,0,0) + if self.h != self.w: + x, pad = square_pad(x) + [n, c, h, w] = x.size() + mask = torch.triu(torch.ones_like(x)) + if self.w <= self.h: #default: main slice at 12 o-clock pointing down + mask = TF.rotate(mask, (360/(num_rays*2)) - 45) + masked = mask * x + pizza_slice = torch.concat((masked[:, :, :, :w//2], TF.hflip(masked[:, :, :, :w//2])), -1) + else: #if landscape format, main slice at 9 o-clock pointing right + mask = TF.rotate(mask, (360/(num_rays*2)) + 45) + masked = mask * x + pizza_slice = torch.concat((TF.vflip(masked[:, :, h//2:, :]), masked[:, :, h//2:, :]), -2) + pizza = torch.zeros_like(x) + for i in range(num_rays): + pizza += TF.rotate(pizza_slice, i*(360/num_rays)) + if self.h != self.w: + pizza = remove_pad(pizza, pad) + return pizza + def symmetry_transformation_fn(x): + [n, c, h, w] = x.size() + if len(args.override_str) > 0 and args.override_str != ".": + #using override string + func_dict = {"/":"diagonal_pos", + "\\":"diagonal_neg", + "-":"vertical", + "|":"horizontal", + "*":"radial"} + m = globals()['SymmTransforms'](x.size()) + for char in args.override_str: + if char == ".": + continue + t_func = getattr(m, func_dict[char]) + if char == "*": + x = t_func(x, args.n_rays) + else: + x = t_func(x) + else: + #if no override, using boolean parameters + symm_t = SymmTransforms(x.size()) + if args.use_diagonal_pos_symmetry: + x = symm_t.diagonal_pos(x) + if args.use_diagonal_neg_symmetry: + x = symm_t.diagonal_neg(x) if args.use_horizontal_symmetry: - [n, c, h, w] = x.size() - x = torch.concat((x[:, :, :, :w//2], torch.flip(x[:, :, :, :w//2], [-1])), -1) - print("horizontal symmetry applied") + x = symm_t.horizontal(x) if args.use_vertical_symmetry: - [n, c, h, w] = x.size() - x = torch.concat((x[:, :, :h//2, :], torch.flip(x[:, :, :h//2, :], [-2])), -2) - print("vertical symmetry applied") - return x + x = symm_t.vertical(x) + if args.use_radial_symmetry: + x = symm_t.radial(x, args.n_rays) + return x def do_run(): seed = args.seed @@ -1620,8 +1718,13 @@ def save_settings(): 'turbo_mode':turbo_mode, 'turbo_steps':turbo_steps, 'turbo_preroll':turbo_preroll, - 'use_horizontal_symmetry':use_horizontal_symmetry, - 'use_vertical_symmetry':use_vertical_symmetry, + 'use_horizontal_symmetry': use_horizontal_symmetry, + 'use_vertical_symmetry': use_vertical_symmetry, + 'use_diagonal_pos_symmetry': use_diagonal_pos_symmetry, + 'use_diagonal_neg_symmetry': use_diagonal_neg_symmetry, + 'use_radial_symmetry': use_radial_symmetry, + 'n_rays': n_rays, + 'override_str':override_str, 'transformation_percent':transformation_percent, #video init settings 'video_init_steps': video_init_steps, @@ -2116,7 +2219,6 @@ def download_model(diffusion_model_name, uri_index=0): batchFolder = f'{outDirPath}/{batch_name}' createPath(batchFolder) - # %% # !! {"metadata":{ # !! "id": "AnimSetTop" @@ -2662,7 +2764,6 @@ def warp(frame1, frame2, flo_path, blend=0.5, weights_path=None): os.chdir(PROJECT_DIR) - # %% # !! {"metadata":{ # !! "id": "FlowFns2" @@ -2819,12 +2920,21 @@ def warp(frame1, frame2, flo_path, blend=0.5, weights_path=None): cut_icgray_p = watercolor_cut_icgray_p #@markdown --- +#@markdown diagonal symmetry requires a square image. n_rays must be at least 1 (for radial symmetry). #@markdown ####**Transformation Settings:** -use_vertical_symmetry = False #@param {type:"boolean"} +use_diagonal_pos_symmetry = False #@param {type:"boolean"} +use_diagonal_neg_symmetry = False #@param {type:"boolean"} use_horizontal_symmetry = False #@param {type:"boolean"} +use_vertical_symmetry = False #@param {type:"boolean"} +use_radial_symmetry = False #@param {type:"boolean"} +n_rays = 1 #@param{type: 'number'} transformation_percent = [0.09] #@param +### ADVANCED SYMMETRY: +# override_str can be used to change the order of transforms, For example r"|-/\*." means to apply horizontal (|), vertical (-), diagonal+ (/), diagonal- (\), radial (*). +# note, override_str must end in a '.' +override_str = r"." # %% # !! {"metadata":{ @@ -3028,8 +3138,13 @@ def move_files(start_num, end_num, old_folder, new_folder): 'turbo_mode':turbo_mode, 'turbo_steps':turbo_steps, 'turbo_preroll':turbo_preroll, - 'use_vertical_symmetry': use_vertical_symmetry, 'use_horizontal_symmetry': use_horizontal_symmetry, + 'use_vertical_symmetry': use_vertical_symmetry, + 'use_diagonal_pos_symmetry': use_diagonal_pos_symmetry, + 'use_diagonal_neg_symmetry': use_diagonal_neg_symmetry, + 'use_radial_symmetry': use_radial_symmetry, + 'n_rays': n_rays, + 'override_str':override_str, 'transformation_percent': transformation_percent, #video init settings 'video_init_steps': video_init_steps, @@ -3096,8 +3211,8 @@ def move_files(start_num, end_num, old_folder, new_folder): # %% # !! {"metadata":{ -# !! "id": "CreateVid", -# !! "cellView": "form" +# !! "cellView": "form", +# !! "id": "CreateVid" # !! }} import PIL # @title ### **Create video** @@ -3227,8 +3342,8 @@ def move_files(start_num, end_num, old_folder, new_folder): # %% # !! {"main_metadata":{ -# !! "anaconda-cloud": {}, # !! "accelerator": "GPU", +# !! "anaconda-cloud": {}, # !! "colab": { # !! "collapsed_sections": [ # !! "CreditsChTop", @@ -3246,14 +3361,14 @@ def move_files(start_num, end_num, old_folder, new_folder): # !! "FlowFns1", # !! "FlowFns2" # !! ], +# !! "include_colab_link": true, # !! "machine_shape": "hm", # !! "name": "Disco Diffusion v5.6 [Now with portrait_generator_v001]", # !! "private_outputs": true, -# !! "provenance": [], -# !! "include_colab_link": true +# !! "provenance": [] # !! }, # !! "kernelspec": { -# !! "display_name": "Python 3", +# !! "display_name": "Python 3 (ipykernel)", # !! "language": "python", # !! "name": "python3" # !! }, @@ -3267,6 +3382,6 @@ def move_files(start_num, end_num, old_folder, new_folder): # !! "name": "python", # !! "nbconvert_exporter": "python", # !! "pygments_lexer": "ipython3", -# !! "version": "3.6.1" +# !! "version": "3.9.12" # !! } # !! }}