mirror of
https://github.com/comfyanonymous/ComfyUI.git
synced 2025-06-08 15:17:14 +00:00
* Add Ideogram generate node. * Add staging api. * Add API_NODE and common error for missing auth token (#5) * Add Minimax Video Generation + Async Task queue polling example (#6) * [Minimax] Show video preview and embed workflow in ouput (#7) * Remove uv.lock * Remove polling operations. * Revert "Remove polling operations." * Update stubs. * Added Ideogram and Minimax back in. * Added initial BFL Flux 1.1 [pro] Ultra node (#11) * Add --comfy-api-base launch arg (#13) * Add instructions for staging development. (#14) * remove validation to make it easier to run against LAN copies of the API * Manually add BFL polling status response schema (#15) * Add function for uploading files. (#18) * Add Luma nodes (#16) * Refactor util functions (#20) * Add VIDEO type (#21) * Add rest of Luma node functionality (#19) * Fix image_luma_ref not working (#28) * [Bug] Remove duplicated option T2V-01 in MinimaxTextToVideoNode (#31) * Add utils to map from pydantic model fields to comfy node inputs (#30) * add veo2, bump av req (#32) * Add Recraft nodes (#29) * Add Kling Nodes (#12) * Add Camera Concepts (luma_concepts) to Luma Video nodes (#33) * Add Runway nodes (#17) * Convert Minimax node to use VIDEO output type (#34) * Standard `CATEGORY` system for api nodes (#35) * Set `Content-Type` header when uploading files (#36) * add better error propagation to veo2 (#37) * Add Realistic Image and Logo Raster styles for Recraft v3 (#38) * Fix runway image upload and progress polling (#39) * Fix image upload for Luma: only include `Content-Type` header field if it's set explicitly (#40) * Moved Luma nodes to nodes_luma.py (#47) * Moved Recraft nodes to nodes_recraft.py (#48) * Add Pixverse nodes (#46) * Move and fix BFL nodes to node_bfl.py (#49) * Move and edit Minimax node to nodes_minimax.py (#50) * Add Minimax Image to Video node + Cleanup (#51) * Add Recraft Text to Vector node, add Save SVG node to handle its output (#53) * Added pixverse_template support to Pixverse Text to Video node (#54) * Added Recraft Controls + Recraft Color RGB nodes (#57) * split remaining nodes out of nodes_api, make utility lib, refactor ideogram (#61) * Add types and doctstrings to utils file (#64) * Fix: `PollingOperation` progress bar update progress by absolute value (#65) * Use common download function in kling nodes module (#67) * Fix: Luma video nodes in `api nodes/image` category (#68) * Set request type explicitly (#66) * Add `control_after_generate` to all seed inputs (#69) * Fix bug: deleting `Content-Type` when property does not exist (#73) * Add preview to Save SVG node (#74) * change default poll interval (#76), rework veo2 * Add Pixverse and updated Kling types (#75) * Added Pixverse Image to VIdeo node (#77) * Add Pixverse Transition Video node (#79) * Proper ray-1-6 support as fix has been applied in backend (#80) * Added Recraft Style - Infinite Style Library node (#82) * add ideogram v3 (#83) * [Kling] Split Camera Control config to its own node (#81) * Add Pika i2v and t2v nodes (#52) * Temporary Fix for Runway (#87) * Added Stability Stable Image Ultra node (#86) * Remove Runway nodes (#88) * Fix: Prompt text can't be validated in Kling nodes when using primitive nodes (#90) * Fix: typo in node name "Stabiliy" => "Stability" (#91) * Add String (Multiline) node (#93) * Update Pika Duration and Resolution options (#94) * Change base branch to master. Not main. (#95) * Fix UploadRequest file_name param (#98) * Removed Infinite Style Library until later (#99) * fix ideogram style types (#100) * fix multi image return (#101) * add metadata saving to SVG (#102) * Bump templates version to include API node template workflows (#104) * Fix: `download_url_to_video_output` return type (#103) * fix 4o generation bug (#106) * Serve SVG files directly (#107) * Add a bunch of nodes, 3 ready to use, the rest waiting for endpoint support (#108) * Revert "Serve SVG files directly" (#111) * Expose 4 remaining Recraft nodes (#112) * [Kling] Add `Duration` and `Video ID` outputs (#105) * Fix: datamodel-codegen sets string#binary type to non-existent `bytes_aliased` variable (#114) * Fix: Dall-e 2 not setting request content-type dynamically (#113) * Default request timeout: one hour. (#116) * Add Kling nodes: camera control, start-end frame, lip-sync, video extend (#115) * Add 8 nodes - 4 BFL, 4 Stability (#117) * Fix error for Recraft ImageToImage error for nonexistent random_seed param (#118) * Add remaining Pika nodes (#119) * Make controls input work for Recraft Image to Image node (#120) * Use upstream PR: Support saving Comfy VIDEO type to buffer (#123) * Use Upstream PR: "Fix: Error creating video when sliced audio tensor chunks are non-c-contiguous" (#127) * Improve audio upload utils (#128) * Fix: Nested `AnyUrl` in request model cannot be serialized (Kling, Runway) (#129) * Show errors and API output URLs to the user (change log levels) (#131) * Fix: Luma I2I fails when weight is <=0.01 (#132) * Change category of `LumaConcepts` node from image to video (#133) * Fix: `image.shape` accessed before `image` is null-checked (#134) * Apply small fixes and most prompt validation (if needed to avoid API error) (#135) * Node name/category modifications (#140) * Add back Recraft Style - Infinite Style Library node (#141) * Fixed Kling: Check attributes of pydantic types. (#144) * Bump `comfyui-workflow-templates` version (#142) * [Kling] Print response data when error validating response (#146) * Fix: error validating Kling image response, trying to use `"key" in` on Pydantic class instance (#147) * [Kling] Fix: Correct/verify supported subset of input combos in Kling nodes (#149) * [Kling] Fix typo in node description (#150) * [Kling] Fix: CFG min/max not being enforced (#151) * Rebase launch-rebase (private) on prep-branch (public copy of master) (#153) * Bump templates version (#154) * Fix: Kling image gen nodes don't return entire batch when `n` > 1 (#152) * Remove pixverse_template from PixVerse Transition Video node (#155) * Invert image_weight value on Luma Image to Image node (#156) * Invert and resize mask for Ideogram V3 node to match masking conventions (#158) * [Kling] Fix: image generation nodes not returning Tuple (#159) * [Bug] [Kling] Fix Kling camera control (#161) * Kling Image Gen v2 + improve node descriptions for Flux/OpenAI (#160) * [Kling] Don't return video_id from dual effect video (#162) * Bump frontend to 1.18.8 (#163) * Use 3.9 compat syntax (#164) * Use Python 3.10 * add example env var * Update templates to 0.1.11 * Bump frontend to 1.18.9 --------- Co-authored-by: Robin Huang <robin.j.huang@gmail.com> Co-authored-by: Christian Byrne <cbyrne@comfy.org> Co-authored-by: thot experiment <94414189+thot-experiment@users.noreply.github.com>
750 lines
25 KiB
Python
750 lines
25 KiB
Python
"""
|
||
Pika x ComfyUI API Nodes
|
||
|
||
Pika API docs: https://pika-827374fb.mintlify.app/api-reference
|
||
"""
|
||
|
||
import io
|
||
from typing import Optional, TypeVar
|
||
import logging
|
||
import torch
|
||
import numpy as np
|
||
from comfy_api_nodes.apis import (
|
||
PikaBodyGenerate22T2vGenerate22T2vPost,
|
||
PikaGenerateResponse,
|
||
PikaBodyGenerate22I2vGenerate22I2vPost,
|
||
PikaVideoResponse,
|
||
PikaBodyGenerate22C2vGenerate22PikascenesPost,
|
||
IngredientsMode,
|
||
PikaDurationEnum,
|
||
PikaResolutionEnum,
|
||
PikaBodyGeneratePikaffectsGeneratePikaffectsPost,
|
||
PikaBodyGeneratePikadditionsGeneratePikadditionsPost,
|
||
PikaBodyGeneratePikaswapsGeneratePikaswapsPost,
|
||
PikaBodyGenerate22KeyframeGenerate22PikaframesPost,
|
||
Pikaffect,
|
||
)
|
||
from comfy_api_nodes.apis.client import (
|
||
ApiEndpoint,
|
||
HttpMethod,
|
||
SynchronousOperation,
|
||
PollingOperation,
|
||
EmptyRequest,
|
||
)
|
||
from comfy_api_nodes.apinode_utils import (
|
||
tensor_to_bytesio,
|
||
download_url_to_video_output,
|
||
)
|
||
from comfy_api_nodes.mapper_utils import model_field_to_node_input
|
||
from comfy_api.input_impl.video_types import VideoInput, VideoContainer, VideoCodec
|
||
from comfy_api.input_impl import VideoFromFile
|
||
from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeOptions
|
||
|
||
R = TypeVar("R")
|
||
|
||
PATH_PIKADDITIONS = "/proxy/pika/generate/pikadditions"
|
||
PATH_PIKASWAPS = "/proxy/pika/generate/pikaswaps"
|
||
PATH_PIKAFFECTS = "/proxy/pika/generate/pikaffects"
|
||
|
||
PIKA_API_VERSION = "2.2"
|
||
PATH_TEXT_TO_VIDEO = f"/proxy/pika/generate/{PIKA_API_VERSION}/t2v"
|
||
PATH_IMAGE_TO_VIDEO = f"/proxy/pika/generate/{PIKA_API_VERSION}/i2v"
|
||
PATH_PIKAFRAMES = f"/proxy/pika/generate/{PIKA_API_VERSION}/pikaframes"
|
||
PATH_PIKASCENES = f"/proxy/pika/generate/{PIKA_API_VERSION}/pikascenes"
|
||
|
||
PATH_VIDEO_GET = "/proxy/pika/videos"
|
||
|
||
|
||
class PikaApiError(Exception):
|
||
"""Exception for Pika API errors."""
|
||
|
||
pass
|
||
|
||
|
||
def is_valid_video_response(response: PikaVideoResponse) -> bool:
|
||
"""Check if the video response is valid."""
|
||
return hasattr(response, "url") and response.url is not None
|
||
|
||
|
||
def is_valid_initial_response(response: PikaGenerateResponse) -> bool:
|
||
"""Check if the initial response is valid."""
|
||
return hasattr(response, "video_id") and response.video_id is not None
|
||
|
||
|
||
class PikaNodeBase(ComfyNodeABC):
|
||
"""Base class for Pika nodes."""
|
||
|
||
@classmethod
|
||
def get_base_inputs_types(
|
||
cls, request_model
|
||
) -> dict[str, tuple[IO, InputTypeOptions]]:
|
||
"""Get the base required inputs types common to all Pika nodes."""
|
||
return {
|
||
"prompt_text": model_field_to_node_input(
|
||
IO.STRING,
|
||
request_model,
|
||
"promptText",
|
||
multiline=True,
|
||
),
|
||
"negative_prompt": model_field_to_node_input(
|
||
IO.STRING,
|
||
request_model,
|
||
"negativePrompt",
|
||
multiline=True,
|
||
),
|
||
"seed": model_field_to_node_input(
|
||
IO.INT,
|
||
request_model,
|
||
"seed",
|
||
min=0,
|
||
max=0xFFFFFFFF,
|
||
control_after_generate=True,
|
||
),
|
||
"resolution": model_field_to_node_input(
|
||
IO.COMBO,
|
||
request_model,
|
||
"resolution",
|
||
enum_type=PikaResolutionEnum,
|
||
),
|
||
"duration": model_field_to_node_input(
|
||
IO.COMBO,
|
||
request_model,
|
||
"duration",
|
||
enum_type=PikaDurationEnum,
|
||
),
|
||
}
|
||
|
||
CATEGORY = "api node/video/Pika"
|
||
API_NODE = True
|
||
FUNCTION = "api_call"
|
||
RETURN_TYPES = ("VIDEO",)
|
||
|
||
def poll_for_task_status(
|
||
self, task_id: str, auth_token: str
|
||
) -> PikaGenerateResponse:
|
||
polling_operation = PollingOperation(
|
||
poll_endpoint=ApiEndpoint(
|
||
path=f"{PATH_VIDEO_GET}/{task_id}",
|
||
method=HttpMethod.GET,
|
||
request_model=EmptyRequest,
|
||
response_model=PikaVideoResponse,
|
||
),
|
||
completed_statuses=[
|
||
"finished",
|
||
],
|
||
failed_statuses=["failed", "cancelled"],
|
||
status_extractor=lambda response: (
|
||
response.status.value if response.status else None
|
||
),
|
||
progress_extractor=lambda response: (
|
||
response.progress if hasattr(response, "progress") else None
|
||
),
|
||
auth_token=auth_token,
|
||
)
|
||
return polling_operation.execute()
|
||
|
||
def execute_task(
|
||
self,
|
||
initial_operation: SynchronousOperation[R, PikaGenerateResponse],
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
"""Executes the initial operation then polls for the task status until it is completed.
|
||
|
||
Args:
|
||
initial_operation: The initial operation to execute.
|
||
auth_token: The authentication token to use for the API call.
|
||
|
||
Returns:
|
||
A tuple containing the video file as a VIDEO output.
|
||
"""
|
||
initial_response = initial_operation.execute()
|
||
if not is_valid_initial_response(initial_response):
|
||
error_msg = f"Pika initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}"
|
||
logging.error(error_msg)
|
||
raise PikaApiError(error_msg)
|
||
|
||
task_id = initial_response.video_id
|
||
final_response = self.poll_for_task_status(task_id, auth_token)
|
||
if not is_valid_video_response(final_response):
|
||
error_msg = (
|
||
f"Pika task {task_id} succeeded but no video data found in response."
|
||
)
|
||
logging.error(error_msg)
|
||
raise PikaApiError(error_msg)
|
||
|
||
video_url = str(final_response.url)
|
||
logging.info("Pika task %s succeeded. Video URL: %s", task_id, video_url)
|
||
|
||
return (download_url_to_video_output(video_url),)
|
||
|
||
|
||
class PikaImageToVideoV2_2(PikaNodeBase):
|
||
"""Pika 2.2 Image to Video Node."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
return {
|
||
"required": {
|
||
"image": (
|
||
IO.IMAGE,
|
||
{"tooltip": "The image to convert to video"},
|
||
),
|
||
**cls.get_base_inputs_types(PikaBodyGenerate22I2vGenerate22I2vPost),
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Sends an image and prompt to the Pika API v2.2 to generate a video."
|
||
|
||
def api_call(
|
||
self,
|
||
image: torch.Tensor,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
resolution: str,
|
||
duration: int,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
# Convert image to BytesIO
|
||
image_bytes_io = tensor_to_bytesio(image)
|
||
image_bytes_io.seek(0)
|
||
|
||
pika_files = {"image": ("image.png", image_bytes_io, "image/png")}
|
||
|
||
# Prepare non-file data
|
||
pika_request_data = PikaBodyGenerate22I2vGenerate22I2vPost(
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
resolution=resolution,
|
||
duration=duration,
|
||
)
|
||
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_IMAGE_TO_VIDEO,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGenerate22I2vGenerate22I2vPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=pika_request_data,
|
||
files=pika_files,
|
||
content_type="multipart/form-data",
|
||
auth_token=auth_token,
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
class PikaTextToVideoNodeV2_2(PikaNodeBase):
|
||
"""Pika Text2Video v2.2 Node."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
return {
|
||
"required": {
|
||
**cls.get_base_inputs_types(PikaBodyGenerate22T2vGenerate22T2vPost),
|
||
"aspect_ratio": model_field_to_node_input(
|
||
IO.FLOAT,
|
||
PikaBodyGenerate22T2vGenerate22T2vPost,
|
||
"aspectRatio",
|
||
step=0.001,
|
||
min=0.4,
|
||
max=2.5,
|
||
default=1.7777777777777777,
|
||
),
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Sends a text prompt to the Pika API v2.2 to generate a video."
|
||
|
||
def api_call(
|
||
self,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
resolution: str,
|
||
duration: int,
|
||
aspect_ratio: float,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_TEXT_TO_VIDEO,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGenerate22T2vGenerate22T2vPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=PikaBodyGenerate22T2vGenerate22T2vPost(
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
resolution=resolution,
|
||
duration=duration,
|
||
aspectRatio=aspect_ratio,
|
||
),
|
||
auth_token=auth_token,
|
||
content_type="application/x-www-form-urlencoded",
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
class PikaScenesV2_2(PikaNodeBase):
|
||
"""PikaScenes v2.2 Node."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
image_ingredient_input = (
|
||
IO.IMAGE,
|
||
{"tooltip": "Image that will be used as ingredient to create a video."},
|
||
)
|
||
return {
|
||
"required": {
|
||
**cls.get_base_inputs_types(
|
||
PikaBodyGenerate22C2vGenerate22PikascenesPost,
|
||
),
|
||
"ingredients_mode": model_field_to_node_input(
|
||
IO.COMBO,
|
||
PikaBodyGenerate22C2vGenerate22PikascenesPost,
|
||
"ingredientsMode",
|
||
enum_type=IngredientsMode,
|
||
default="creative",
|
||
),
|
||
"aspect_ratio": model_field_to_node_input(
|
||
IO.FLOAT,
|
||
PikaBodyGenerate22C2vGenerate22PikascenesPost,
|
||
"aspectRatio",
|
||
step=0.001,
|
||
min=0.4,
|
||
max=2.5,
|
||
default=1.7777777777777777,
|
||
),
|
||
},
|
||
"optional": {
|
||
"image_ingredient_1": image_ingredient_input,
|
||
"image_ingredient_2": image_ingredient_input,
|
||
"image_ingredient_3": image_ingredient_input,
|
||
"image_ingredient_4": image_ingredient_input,
|
||
"image_ingredient_5": image_ingredient_input,
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Combine your images to create a video with the objects in them. Upload multiple images as ingredients and generate a high-quality video that incorporates all of them."
|
||
|
||
def api_call(
|
||
self,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
resolution: str,
|
||
duration: int,
|
||
ingredients_mode: str,
|
||
aspect_ratio: float,
|
||
image_ingredient_1: Optional[torch.Tensor] = None,
|
||
image_ingredient_2: Optional[torch.Tensor] = None,
|
||
image_ingredient_3: Optional[torch.Tensor] = None,
|
||
image_ingredient_4: Optional[torch.Tensor] = None,
|
||
image_ingredient_5: Optional[torch.Tensor] = None,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
# Convert all passed images to BytesIO
|
||
all_image_bytes_io = []
|
||
for image in [
|
||
image_ingredient_1,
|
||
image_ingredient_2,
|
||
image_ingredient_3,
|
||
image_ingredient_4,
|
||
image_ingredient_5,
|
||
]:
|
||
if image is not None:
|
||
image_bytes_io = tensor_to_bytesio(image)
|
||
image_bytes_io.seek(0)
|
||
all_image_bytes_io.append(image_bytes_io)
|
||
|
||
pika_files = [
|
||
("images", (f"image_{i}.png", image_bytes_io, "image/png"))
|
||
for i, image_bytes_io in enumerate(all_image_bytes_io)
|
||
]
|
||
|
||
pika_request_data = PikaBodyGenerate22C2vGenerate22PikascenesPost(
|
||
ingredientsMode=ingredients_mode,
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
resolution=resolution,
|
||
duration=duration,
|
||
aspectRatio=aspect_ratio,
|
||
)
|
||
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_PIKASCENES,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGenerate22C2vGenerate22PikascenesPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=pika_request_data,
|
||
files=pika_files,
|
||
content_type="multipart/form-data",
|
||
auth_token=auth_token,
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
class PikAdditionsNode(PikaNodeBase):
|
||
"""Pika Pikadditions Node. Add an image into a video."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
return {
|
||
"required": {
|
||
"video": (IO.VIDEO, {"tooltip": "The video to add an image to."}),
|
||
"image": (IO.IMAGE, {"tooltip": "The image to add to the video."}),
|
||
"prompt_text": model_field_to_node_input(
|
||
IO.STRING,
|
||
PikaBodyGeneratePikadditionsGeneratePikadditionsPost,
|
||
"promptText",
|
||
multiline=True,
|
||
),
|
||
"negative_prompt": model_field_to_node_input(
|
||
IO.STRING,
|
||
PikaBodyGeneratePikadditionsGeneratePikadditionsPost,
|
||
"negativePrompt",
|
||
multiline=True,
|
||
),
|
||
"seed": model_field_to_node_input(
|
||
IO.INT,
|
||
PikaBodyGeneratePikadditionsGeneratePikadditionsPost,
|
||
"seed",
|
||
min=0,
|
||
max=0xFFFFFFFF,
|
||
control_after_generate=True,
|
||
),
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Add any object or image into your video. Upload a video and specify what you’d like to add to create a seamlessly integrated result."
|
||
|
||
def api_call(
|
||
self,
|
||
video: VideoInput,
|
||
image: torch.Tensor,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
# Convert video to BytesIO
|
||
video_bytes_io = io.BytesIO()
|
||
video.save_to(video_bytes_io, format=VideoContainer.MP4, codec=VideoCodec.H264)
|
||
video_bytes_io.seek(0)
|
||
|
||
# Convert image to BytesIO
|
||
image_bytes_io = tensor_to_bytesio(image)
|
||
image_bytes_io.seek(0)
|
||
|
||
pika_files = [
|
||
("video", ("video.mp4", video_bytes_io, "video/mp4")),
|
||
("image", ("image.png", image_bytes_io, "image/png")),
|
||
]
|
||
|
||
# Prepare non-file data
|
||
pika_request_data = PikaBodyGeneratePikadditionsGeneratePikadditionsPost(
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
)
|
||
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_PIKADDITIONS,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGeneratePikadditionsGeneratePikadditionsPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=pika_request_data,
|
||
files=pika_files,
|
||
content_type="multipart/form-data",
|
||
auth_token=auth_token,
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
class PikaSwapsNode(PikaNodeBase):
|
||
"""Pika Pikaswaps Node."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
return {
|
||
"required": {
|
||
"video": (IO.VIDEO, {"tooltip": "The video to swap an object in."}),
|
||
"image": (
|
||
IO.IMAGE,
|
||
{
|
||
"tooltip": "The image used to replace the masked object in the video."
|
||
},
|
||
),
|
||
"mask": (
|
||
IO.MASK,
|
||
{"tooltip": "Use the mask to define areas in the video to replace"},
|
||
),
|
||
"prompt_text": model_field_to_node_input(
|
||
IO.STRING,
|
||
PikaBodyGeneratePikaswapsGeneratePikaswapsPost,
|
||
"promptText",
|
||
multiline=True,
|
||
),
|
||
"negative_prompt": model_field_to_node_input(
|
||
IO.STRING,
|
||
PikaBodyGeneratePikaswapsGeneratePikaswapsPost,
|
||
"negativePrompt",
|
||
multiline=True,
|
||
),
|
||
"seed": model_field_to_node_input(
|
||
IO.INT,
|
||
PikaBodyGeneratePikaswapsGeneratePikaswapsPost,
|
||
"seed",
|
||
min=0,
|
||
max=0xFFFFFFFF,
|
||
control_after_generate=True,
|
||
),
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Swap out any object or region of your video with a new image or object. Define areas to replace either with a mask or coordinates."
|
||
RETURN_TYPES = ("VIDEO",)
|
||
|
||
def api_call(
|
||
self,
|
||
video: VideoInput,
|
||
image: torch.Tensor,
|
||
mask: torch.Tensor,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
# Convert video to BytesIO
|
||
video_bytes_io = io.BytesIO()
|
||
video.save_to(video_bytes_io, format=VideoContainer.MP4, codec=VideoCodec.H264)
|
||
video_bytes_io.seek(0)
|
||
|
||
# Convert mask to binary mask with three channels
|
||
mask = torch.round(mask)
|
||
mask = mask.repeat(1, 3, 1, 1)
|
||
|
||
# Convert 3-channel binary mask to BytesIO
|
||
mask_bytes_io = io.BytesIO()
|
||
mask_bytes_io.write(mask.numpy().astype(np.uint8))
|
||
mask_bytes_io.seek(0)
|
||
|
||
# Convert image to BytesIO
|
||
image_bytes_io = tensor_to_bytesio(image)
|
||
image_bytes_io.seek(0)
|
||
|
||
pika_files = [
|
||
("video", ("video.mp4", video_bytes_io, "video/mp4")),
|
||
("image", ("image.png", image_bytes_io, "image/png")),
|
||
("modifyRegionMask", ("mask.png", mask_bytes_io, "image/png")),
|
||
]
|
||
|
||
# Prepare non-file data
|
||
pika_request_data = PikaBodyGeneratePikaswapsGeneratePikaswapsPost(
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
)
|
||
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_PIKADDITIONS,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGeneratePikadditionsGeneratePikadditionsPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=pika_request_data,
|
||
files=pika_files,
|
||
content_type="multipart/form-data",
|
||
auth_token=auth_token,
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
class PikaffectsNode(PikaNodeBase):
|
||
"""Pika Pikaffects Node."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
return {
|
||
"required": {
|
||
"image": (
|
||
IO.IMAGE,
|
||
{"tooltip": "The reference image to apply the Pikaffect to."},
|
||
),
|
||
"pikaffect": model_field_to_node_input(
|
||
IO.COMBO,
|
||
PikaBodyGeneratePikaffectsGeneratePikaffectsPost,
|
||
"pikaffect",
|
||
enum_type=Pikaffect,
|
||
default="Cake-ify",
|
||
),
|
||
"prompt_text": model_field_to_node_input(
|
||
IO.STRING,
|
||
PikaBodyGeneratePikaffectsGeneratePikaffectsPost,
|
||
"promptText",
|
||
multiline=True,
|
||
),
|
||
"negative_prompt": model_field_to_node_input(
|
||
IO.STRING,
|
||
PikaBodyGeneratePikaffectsGeneratePikaffectsPost,
|
||
"negativePrompt",
|
||
multiline=True,
|
||
),
|
||
"seed": model_field_to_node_input(
|
||
IO.INT,
|
||
PikaBodyGeneratePikaffectsGeneratePikaffectsPost,
|
||
"seed",
|
||
min=0,
|
||
max=0xFFFFFFFF,
|
||
control_after_generate=True,
|
||
),
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Generate a video with a specific Pikaffect. Supported Pikaffects: Cake-ify, Crumble, Crush, Decapitate, Deflate, Dissolve, Explode, Eye-pop, Inflate, Levitate, Melt, Peel, Poke, Squish, Ta-da, Tear"
|
||
|
||
def api_call(
|
||
self,
|
||
image: torch.Tensor,
|
||
pikaffect: str,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_PIKAFFECTS,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGeneratePikaffectsGeneratePikaffectsPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=PikaBodyGeneratePikaffectsGeneratePikaffectsPost(
|
||
pikaffect=pikaffect,
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
),
|
||
files={"image": ("image.png", tensor_to_bytesio(image), "image/png")},
|
||
content_type="multipart/form-data",
|
||
auth_token=auth_token,
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
class PikaStartEndFrameNode2_2(PikaNodeBase):
|
||
"""PikaFrames v2.2 Node."""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
return {
|
||
"required": {
|
||
"image_start": (IO.IMAGE, {"tooltip": "The first image to combine."}),
|
||
"image_end": (IO.IMAGE, {"tooltip": "The last image to combine."}),
|
||
**cls.get_base_inputs_types(
|
||
PikaBodyGenerate22KeyframeGenerate22PikaframesPost
|
||
),
|
||
},
|
||
"hidden": {
|
||
"auth_token": "AUTH_TOKEN_COMFY_ORG",
|
||
},
|
||
}
|
||
|
||
DESCRIPTION = "Generate a video by combining your first and last frame. Upload two images to define the start and end points, and let the AI create a smooth transition between them."
|
||
|
||
def api_call(
|
||
self,
|
||
image_start: torch.Tensor,
|
||
image_end: torch.Tensor,
|
||
prompt_text: str,
|
||
negative_prompt: str,
|
||
seed: int,
|
||
resolution: str,
|
||
duration: int,
|
||
auth_token: Optional[str] = None,
|
||
) -> tuple[VideoFromFile]:
|
||
|
||
pika_files = [
|
||
(
|
||
"keyFrames",
|
||
("image_start.png", tensor_to_bytesio(image_start), "image/png"),
|
||
),
|
||
("keyFrames", ("image_end.png", tensor_to_bytesio(image_end), "image/png")),
|
||
]
|
||
|
||
initial_operation = SynchronousOperation(
|
||
endpoint=ApiEndpoint(
|
||
path=PATH_PIKAFRAMES,
|
||
method=HttpMethod.POST,
|
||
request_model=PikaBodyGenerate22KeyframeGenerate22PikaframesPost,
|
||
response_model=PikaGenerateResponse,
|
||
),
|
||
request=PikaBodyGenerate22KeyframeGenerate22PikaframesPost(
|
||
promptText=prompt_text,
|
||
negativePrompt=negative_prompt,
|
||
seed=seed,
|
||
resolution=resolution,
|
||
duration=duration,
|
||
),
|
||
files=pika_files,
|
||
content_type="multipart/form-data",
|
||
auth_token=auth_token,
|
||
)
|
||
|
||
return self.execute_task(initial_operation, auth_token)
|
||
|
||
|
||
NODE_CLASS_MAPPINGS = {
|
||
"PikaImageToVideoNode2_2": PikaImageToVideoV2_2,
|
||
"PikaTextToVideoNode2_2": PikaTextToVideoNodeV2_2,
|
||
"PikaScenesV2_2": PikaScenesV2_2,
|
||
"Pikadditions": PikAdditionsNode,
|
||
"Pikaswaps": PikaSwapsNode,
|
||
"Pikaffects": PikaffectsNode,
|
||
"PikaStartEndFrameNode2_2": PikaStartEndFrameNode2_2,
|
||
}
|
||
|
||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||
"PikaImageToVideoNode2_2": "Pika Image to Video",
|
||
"PikaTextToVideoNode2_2": "Pika Text to Video",
|
||
"PikaScenesV2_2": "Pika Scenes (Video Image Composition)",
|
||
"Pikadditions": "Pikadditions (Video Object Insertion)",
|
||
"Pikaswaps": "Pika Swaps (Video Object Replacement)",
|
||
"Pikaffects": "Pikaffects (Video Effects)",
|
||
"PikaStartEndFrameNode2_2": "Pika Start and End Frame to Video",
|
||
}
|