diff --git a/comfy_extras/v3/nodes_controlnet.py b/comfy_extras/v3/nodes_controlnet.py new file mode 100644 index 000000000..12d91a1ce --- /dev/null +++ b/comfy_extras/v3/nodes_controlnet.py @@ -0,0 +1,125 @@ +from comfy.cldm.control_types import UNION_CONTROLNET_TYPES +import comfy.utils +from comfy_api.v3 import io + + +class ControlNetApplyAdvanced_V3(io.ComfyNodeV3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="ControlNetApplyAdvanced_V3", + display_name="Apply ControlNet _V3", + category="conditioning/controlnet", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.ControlNet.Input("control_net"), + io.Image.Input("image"), + io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01), + io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001), + io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001), + io.Vae.Input("vae", optional=True), + ], + outputs=[ + io.Conditioning.Output("positive_out", display_name="positive"), + io.Conditioning.Output("negative_out", display_name="negative"), + ], + ) + + @classmethod + def execute(cls, positive, negative, control_net, image, strength, start_percent, end_percent, vae=None, extra_concat=[]) -> io.NodeOutput: + if strength == 0: + return io.NodeOutput(positive, negative) + + control_hint = image.movedim(-1,1) + cnets = {} + + out = [] + for conditioning in [positive, negative]: + c = [] + for t in conditioning: + d = t[1].copy() + + prev_cnet = d.get('control', None) + if prev_cnet in cnets: + c_net = cnets[prev_cnet] + else: + c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent), vae=vae, extra_concat=extra_concat) + c_net.set_previous_controlnet(prev_cnet) + cnets[prev_cnet] = c_net + + d['control'] = c_net + d['control_apply_to_uncond'] = False + n = [t[0], d] + c.append(n) + out.append(c) + return io.NodeOutput(out[0], out[1]) + + +class SetUnionControlNetType_V3(io.ComfyNodeV3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="SetUnionControlNetType_V3", + category="conditioning/controlnet", + inputs=[ + io.ControlNet.Input("control_net"), + io.Combo.Input("type", options=["auto"] + list(UNION_CONTROLNET_TYPES.keys())), + ], + outputs=[ + io.ControlNet.Output("control_net_out"), + ], + ) + + @classmethod + def execute(cls, control_net, type) -> io.NodeOutput: + control_net = control_net.copy() + type_number = UNION_CONTROLNET_TYPES.get(type, -1) + if type_number >= 0: + control_net.set_extra_arg("control_type", [type_number]) + else: + control_net.set_extra_arg("control_type", []) + + return io.NodeOutput(control_net) + + +class ControlNetInpaintingAliMamaApply_V3(ControlNetApplyAdvanced_V3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="ControlNetInpaintingAliMamaApply_V3", + category="conditioning/controlnet", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.ControlNet.Input("control_net"), + io.Vae.Input("vae"), + io.Image.Input("image"), + io.Mask.Input("mask"), + io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01), + io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001), + io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001), + ], + outputs=[ + io.Conditioning.Output("positive_out", display_name="positive"), + io.Conditioning.Output("negative_out", display_name="negative"), + ], + ) + + @classmethod + def execute(cls, positive, negative, control_net, vae, image, mask, strength, start_percent, end_percent) -> io.NodeOutput: + extra_concat = [] + if control_net.concat_mask: + mask = 1.0 - mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])) + mask_apply = comfy.utils.common_upscale(mask, image.shape[2], image.shape[1], "bilinear", "center").round() + image = image * mask_apply.movedim(1, -1).repeat(1, 1, 1, image.shape[3]) + extra_concat = [mask] + + return super().execute(positive, negative, control_net, image, strength, start_percent, end_percent, vae=vae, extra_concat=extra_concat) + + +NODES_LIST: list[type[io.ComfyNodeV3]] = [ + ControlNetApplyAdvanced_V3, + SetUnionControlNetType_V3, + ControlNetInpaintingAliMamaApply_V3, +] diff --git a/comfy_extras/v3/nodes_stable_cascade.py b/comfy_extras/v3/nodes_stable_cascade.py new file mode 100644 index 000000000..36d7e3321 --- /dev/null +++ b/comfy_extras/v3/nodes_stable_cascade.py @@ -0,0 +1,143 @@ +""" + This file is part of ComfyUI. + Copyright (C) 2024 Stability AI + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +import torch +import nodes +import comfy.utils + +from comfy_api.v3 import io + + +class StableCascade_EmptyLatentImage_V3(io.ComfyNodeV3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="StableCascade_EmptyLatentImage_V3", + category="latent/stable_cascade", + inputs=[ + io.Int.Input("width", default=1024,min=256,max=nodes.MAX_RESOLUTION, step=8), + io.Int.Input("height", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8), + io.Int.Input("compression", default=42, min=4, max=128, step=1), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[ + io.Latent.Output("stage_c", display_name="stage_c"), + io.Latent.Output("stage_b", display_name="stage_b"), + ], + ) + + @classmethod + def execute(cls, width, height, compression, batch_size=1): + c_latent = torch.zeros([batch_size, 16, height // compression, width // compression]) + b_latent = torch.zeros([batch_size, 4, height // 4, width // 4]) + return io.NodeOutput({"samples": c_latent}, {"samples": b_latent}) + + +class StableCascade_StageC_VAEEncode_V3(io.ComfyNodeV3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="StableCascade_StageC_VAEEncode_V3", + category="latent/stable_cascade", + inputs=[ + io.Image.Input("image"), + io.Vae.Input("vae"), + io.Int.Input("compression", default=42, min=4, max=128, step=1), + ], + outputs=[ + io.Latent.Output("stage_c", display_name="stage_c"), + io.Latent.Output("stage_b", display_name="stage_b"), + ], + ) + + @classmethod + def execute(cls, image, vae, compression): + width = image.shape[-2] + height = image.shape[-3] + out_width = (width // compression) * vae.downscale_ratio + out_height = (height // compression) * vae.downscale_ratio + + s = comfy.utils.common_upscale(image.movedim(-1,1), out_width, out_height, "bicubic", "center").movedim(1,-1) + + c_latent = vae.encode(s[:,:,:,:3]) + b_latent = torch.zeros([c_latent.shape[0], 4, (height // 8) * 2, (width // 8) * 2]) + return io.NodeOutput({"samples": c_latent}, {"samples": b_latent}) + + +class StableCascade_StageB_Conditioning_V3(io.ComfyNodeV3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="StableCascade_StageB_Conditioning_V3", + category="conditioning/stable_cascade", + inputs=[ + io.Conditioning.Input("conditioning"), + io.Latent.Input("stage_c"), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, conditioning, stage_c): + c = [] + for t in conditioning: + d = t[1].copy() + d['stable_cascade_prior'] = stage_c['samples'] + n = [t[0], d] + c.append(n) + return io.NodeOutput(c) + + +class StableCascade_SuperResolutionControlnet_V3(io.ComfyNodeV3): + @classmethod + def DEFINE_SCHEMA(cls): + return io.SchemaV3( + node_id="StableCascade_SuperResolutionControlnet_V3", + category="_for_testing/stable_cascade", + is_experimental=True, + inputs=[ + io.Image.Input("image"), + io.Vae.Input("vae"), + ], + outputs=[ + io.Image.Output("controlnet_input", display_name="controlnet_input"), + io.Latent.Output("stage_c", display_name="stage_c"), + io.Latent.Output("stage_b", display_name="stage_b"), + ], + ) + + @classmethod + def execute(cls, image, vae): + width = image.shape[-2] + height = image.shape[-3] + batch_size = image.shape[0] + controlnet_input = vae.encode(image[:,:,:,:3]).movedim(1, -1) + + c_latent = torch.zeros([batch_size, 16, height // 16, width // 16]) + b_latent = torch.zeros([batch_size, 4, height // 2, width // 2]) + return io.NodeOutput(controlnet_input, {"samples": c_latent}, {"samples": b_latent}) + + +NODES_LIST: list[type[io.ComfyNodeV3]] = [ + StableCascade_EmptyLatentImage_V3, + StableCascade_StageB_Conditioning_V3, + StableCascade_StageC_VAEEncode_V3, + StableCascade_SuperResolutionControlnet_V3, +] diff --git a/nodes.py b/nodes.py index 4e4dae917..90d20e6a6 100644 --- a/nodes.py +++ b/nodes.py @@ -2299,9 +2299,11 @@ def init_builtin_extra_nodes(): "nodes_tcfg.py", "nodes_v3_test.py", "nodes_v1_test.py", + "v3/nodes_controlnet.py", "v3/nodes_images.py", "v3/nodes_mask.py", "v3/nodes_webcam.py", + "v3/nodes_stable_cascade.py", ] import_failed = []