mirror of
https://github.com/comfyanonymous/ComfyUI.git
synced 2025-09-11 03:58:22 +00:00
add ByteDance video API nodes (#9712)
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import logging
|
import logging
|
||||||
|
import math
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional
|
from typing import Literal, Optional, Type, Union
|
||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -10,28 +11,53 @@ from comfy_api.latest import ComfyExtension, io as comfy_io
|
|||||||
from comfy_api_nodes.util.validation_utils import (
|
from comfy_api_nodes.util.validation_utils import (
|
||||||
validate_image_aspect_ratio_range,
|
validate_image_aspect_ratio_range,
|
||||||
get_number_of_images,
|
get_number_of_images,
|
||||||
|
validate_image_dimensions,
|
||||||
)
|
)
|
||||||
from comfy_api_nodes.apis.client import (
|
from comfy_api_nodes.apis.client import (
|
||||||
ApiEndpoint,
|
ApiEndpoint,
|
||||||
|
EmptyRequest,
|
||||||
HttpMethod,
|
HttpMethod,
|
||||||
SynchronousOperation,
|
SynchronousOperation,
|
||||||
|
PollingOperation,
|
||||||
|
T,
|
||||||
|
)
|
||||||
|
from comfy_api_nodes.apinode_utils import (
|
||||||
|
download_url_to_image_tensor,
|
||||||
|
download_url_to_video_output,
|
||||||
|
upload_images_to_comfyapi,
|
||||||
|
validate_string,
|
||||||
|
image_tensor_pair_to_batch,
|
||||||
)
|
)
|
||||||
from comfy_api_nodes.apinode_utils import download_url_to_image_tensor, upload_images_to_comfyapi, validate_string
|
|
||||||
|
|
||||||
|
|
||||||
BYTEPLUS_ENDPOINT = "/proxy/byteplus/api/v3/images/generations"
|
BYTEPLUS_IMAGE_ENDPOINT = "/proxy/byteplus/api/v3/images/generations"
|
||||||
|
|
||||||
|
# Long-running tasks endpoints(e.g., video)
|
||||||
|
BYTEPLUS_TASK_ENDPOINT = "/proxy/byteplus/api/v3/contents/generations/tasks"
|
||||||
|
BYTEPLUS_TASK_STATUS_ENDPOINT = "/proxy/byteplus/api/v3/contents/generations/tasks" # + /{task_id}
|
||||||
|
|
||||||
|
|
||||||
class Text2ImageModelName(str, Enum):
|
class Text2ImageModelName(str, Enum):
|
||||||
seedream3 = "seedream-3-0-t2i-250415"
|
seedream_3 = "seedream-3-0-t2i-250415"
|
||||||
|
|
||||||
|
|
||||||
class Image2ImageModelName(str, Enum):
|
class Image2ImageModelName(str, Enum):
|
||||||
seededit3 = "seededit-3-0-i2i-250628"
|
seededit_3 = "seededit-3-0-i2i-250628"
|
||||||
|
|
||||||
|
|
||||||
|
class Text2VideoModelName(str, Enum):
|
||||||
|
seedance_1_pro = "seedance-1-0-pro-250528"
|
||||||
|
seedance_1_lite = "seedance-1-0-lite-t2v-250428"
|
||||||
|
|
||||||
|
|
||||||
|
class Image2VideoModelName(str, Enum):
|
||||||
|
"""note(August 31): Pro model only supports FirstFrame: https://docs.byteplus.com/en/docs/ModelArk/1520757"""
|
||||||
|
seedance_1_pro = "seedance-1-0-pro-250528"
|
||||||
|
seedance_1_lite = "seedance-1-0-lite-i2v-250428"
|
||||||
|
|
||||||
|
|
||||||
class Text2ImageTaskCreationRequest(BaseModel):
|
class Text2ImageTaskCreationRequest(BaseModel):
|
||||||
model: Text2ImageModelName = Text2ImageModelName.seedream3
|
model: Text2ImageModelName = Text2ImageModelName.seedream_3
|
||||||
prompt: str = Field(...)
|
prompt: str = Field(...)
|
||||||
response_format: Optional[str] = Field("url")
|
response_format: Optional[str] = Field("url")
|
||||||
size: Optional[str] = Field(None)
|
size: Optional[str] = Field(None)
|
||||||
@@ -41,7 +67,7 @@ class Text2ImageTaskCreationRequest(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class Image2ImageTaskCreationRequest(BaseModel):
|
class Image2ImageTaskCreationRequest(BaseModel):
|
||||||
model: Image2ImageModelName = Image2ImageModelName.seededit3
|
model: Image2ImageModelName = Image2ImageModelName.seededit_3
|
||||||
prompt: str = Field(...)
|
prompt: str = Field(...)
|
||||||
response_format: Optional[str] = Field("url")
|
response_format: Optional[str] = Field("url")
|
||||||
image: str = Field(..., description="Base64 encoded string or image URL")
|
image: str = Field(..., description="Base64 encoded string or image URL")
|
||||||
@@ -58,6 +84,52 @@ class ImageTaskCreationResponse(BaseModel):
|
|||||||
error: dict = Field({}, description="Contains `code` and `message` fields in case of error.")
|
error: dict = Field({}, description="Contains `code` and `message` fields in case of error.")
|
||||||
|
|
||||||
|
|
||||||
|
class TaskTextContent(BaseModel):
|
||||||
|
type: str = Field("text")
|
||||||
|
text: str = Field(...)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskImageContentUrl(BaseModel):
|
||||||
|
url: str = Field(...)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskImageContent(BaseModel):
|
||||||
|
type: str = Field("image_url")
|
||||||
|
image_url: TaskImageContentUrl = Field(...)
|
||||||
|
role: Optional[Literal["first_frame", "last_frame", "reference_image"]] = Field(None)
|
||||||
|
|
||||||
|
|
||||||
|
class Text2VideoTaskCreationRequest(BaseModel):
|
||||||
|
model: Text2VideoModelName = Text2VideoModelName.seedance_1_pro
|
||||||
|
content: list[TaskTextContent] = Field(..., min_length=1)
|
||||||
|
|
||||||
|
|
||||||
|
class Image2VideoTaskCreationRequest(BaseModel):
|
||||||
|
model: Image2VideoModelName = Image2VideoModelName.seedance_1_pro
|
||||||
|
content: list[Union[TaskTextContent, TaskImageContent]] = Field(..., min_length=2)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskCreationResponse(BaseModel):
|
||||||
|
id: str = Field(...)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskStatusError(BaseModel):
|
||||||
|
code: str = Field(...)
|
||||||
|
message: str = Field(...)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskStatusResult(BaseModel):
|
||||||
|
video_url: str = Field(...)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskStatusResponse(BaseModel):
|
||||||
|
id: str = Field(...)
|
||||||
|
model: str = Field(...)
|
||||||
|
status: Literal["queued", "running", "cancelled", "succeeded", "failed"] = Field(...)
|
||||||
|
error: Optional[TaskStatusError] = Field(None)
|
||||||
|
content: Optional[TaskStatusResult] = Field(None)
|
||||||
|
|
||||||
|
|
||||||
RECOMMENDED_PRESETS = [
|
RECOMMENDED_PRESETS = [
|
||||||
("1024x1024 (1:1)", 1024, 1024),
|
("1024x1024 (1:1)", 1024, 1024),
|
||||||
("864x1152 (3:4)", 864, 1152),
|
("864x1152 (3:4)", 864, 1152),
|
||||||
@@ -71,6 +143,25 @@ RECOMMENDED_PRESETS = [
|
|||||||
("Custom", None, None),
|
("Custom", None, None),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# The time in this dictionary are given for 10 seconds duration.
|
||||||
|
VIDEO_TASKS_EXECUTION_TIME = {
|
||||||
|
"seedance-1-0-lite-t2v-250428": {
|
||||||
|
"480p": 40,
|
||||||
|
"720p": 60,
|
||||||
|
"1080p": 90,
|
||||||
|
},
|
||||||
|
"seedance-1-0-lite-i2v-250428": {
|
||||||
|
"480p": 40,
|
||||||
|
"720p": 60,
|
||||||
|
"1080p": 90,
|
||||||
|
},
|
||||||
|
"seedance-1-0-pro-250528": {
|
||||||
|
"480p": 70,
|
||||||
|
"720p": 85,
|
||||||
|
"1080p": 115,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_image_url_from_response(response: ImageTaskCreationResponse) -> str:
|
def get_image_url_from_response(response: ImageTaskCreationResponse) -> str:
|
||||||
if response.error:
|
if response.error:
|
||||||
@@ -81,6 +172,42 @@ def get_image_url_from_response(response: ImageTaskCreationResponse) -> str:
|
|||||||
return response.data[0]["url"]
|
return response.data[0]["url"]
|
||||||
|
|
||||||
|
|
||||||
|
def get_video_url_from_task_status(response: TaskStatusResponse) -> Union[str, None]:
|
||||||
|
"""Returns the video URL from the task status response if it exists."""
|
||||||
|
if hasattr(response, "content") and response.content:
|
||||||
|
return response.content.video_url
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def poll_until_finished(
|
||||||
|
auth_kwargs: dict[str, str],
|
||||||
|
task_id: str,
|
||||||
|
estimated_duration: Optional[int] = None,
|
||||||
|
node_id: Optional[str] = None,
|
||||||
|
) -> TaskStatusResponse:
|
||||||
|
"""Polls the ByteDance API endpoint until the task reaches a terminal state, then returns the response."""
|
||||||
|
return await PollingOperation(
|
||||||
|
poll_endpoint=ApiEndpoint(
|
||||||
|
path=f"{BYTEPLUS_TASK_STATUS_ENDPOINT}/{task_id}",
|
||||||
|
method=HttpMethod.GET,
|
||||||
|
request_model=EmptyRequest,
|
||||||
|
response_model=TaskStatusResponse,
|
||||||
|
),
|
||||||
|
completed_statuses=[
|
||||||
|
"succeeded",
|
||||||
|
],
|
||||||
|
failed_statuses=[
|
||||||
|
"cancelled",
|
||||||
|
"failed",
|
||||||
|
],
|
||||||
|
status_extractor=lambda response: response.status,
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
result_url_extractor=get_video_url_from_task_status,
|
||||||
|
estimated_duration=estimated_duration,
|
||||||
|
node_id=node_id,
|
||||||
|
).execute()
|
||||||
|
|
||||||
|
|
||||||
class ByteDanceImageNode(comfy_io.ComfyNode):
|
class ByteDanceImageNode(comfy_io.ComfyNode):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -94,7 +221,7 @@ class ByteDanceImageNode(comfy_io.ComfyNode):
|
|||||||
comfy_io.Combo.Input(
|
comfy_io.Combo.Input(
|
||||||
"model",
|
"model",
|
||||||
options=[model.value for model in Text2ImageModelName],
|
options=[model.value for model in Text2ImageModelName],
|
||||||
default=Text2ImageModelName.seedream3.value,
|
default=Text2ImageModelName.seedream_3.value,
|
||||||
tooltip="Model name",
|
tooltip="Model name",
|
||||||
),
|
),
|
||||||
comfy_io.String.Input(
|
comfy_io.String.Input(
|
||||||
@@ -203,7 +330,7 @@ class ByteDanceImageNode(comfy_io.ComfyNode):
|
|||||||
}
|
}
|
||||||
response = await SynchronousOperation(
|
response = await SynchronousOperation(
|
||||||
endpoint=ApiEndpoint(
|
endpoint=ApiEndpoint(
|
||||||
path=BYTEPLUS_ENDPOINT,
|
path=BYTEPLUS_IMAGE_ENDPOINT,
|
||||||
method=HttpMethod.POST,
|
method=HttpMethod.POST,
|
||||||
request_model=Text2ImageTaskCreationRequest,
|
request_model=Text2ImageTaskCreationRequest,
|
||||||
response_model=ImageTaskCreationResponse,
|
response_model=ImageTaskCreationResponse,
|
||||||
@@ -227,7 +354,7 @@ class ByteDanceImageEditNode(comfy_io.ComfyNode):
|
|||||||
comfy_io.Combo.Input(
|
comfy_io.Combo.Input(
|
||||||
"model",
|
"model",
|
||||||
options=[model.value for model in Image2ImageModelName],
|
options=[model.value for model in Image2ImageModelName],
|
||||||
default=Image2ImageModelName.seededit3.value,
|
default=Image2ImageModelName.seededit_3.value,
|
||||||
tooltip="Model name",
|
tooltip="Model name",
|
||||||
),
|
),
|
||||||
comfy_io.Image.Input(
|
comfy_io.Image.Input(
|
||||||
@@ -313,7 +440,7 @@ class ByteDanceImageEditNode(comfy_io.ComfyNode):
|
|||||||
)
|
)
|
||||||
response = await SynchronousOperation(
|
response = await SynchronousOperation(
|
||||||
endpoint=ApiEndpoint(
|
endpoint=ApiEndpoint(
|
||||||
path=BYTEPLUS_ENDPOINT,
|
path=BYTEPLUS_IMAGE_ENDPOINT,
|
||||||
method=HttpMethod.POST,
|
method=HttpMethod.POST,
|
||||||
request_model=Image2ImageTaskCreationRequest,
|
request_model=Image2ImageTaskCreationRequest,
|
||||||
response_model=ImageTaskCreationResponse,
|
response_model=ImageTaskCreationResponse,
|
||||||
@@ -324,12 +451,560 @@ class ByteDanceImageEditNode(comfy_io.ComfyNode):
|
|||||||
return comfy_io.NodeOutput(await download_url_to_image_tensor(get_image_url_from_response(response)))
|
return comfy_io.NodeOutput(await download_url_to_image_tensor(get_image_url_from_response(response)))
|
||||||
|
|
||||||
|
|
||||||
|
class ByteDanceTextToVideoNode(comfy_io.ComfyNode):
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return comfy_io.Schema(
|
||||||
|
node_id="ByteDanceTextToVideoNode",
|
||||||
|
display_name="ByteDance Text to Video",
|
||||||
|
category="api node/video/ByteDance",
|
||||||
|
description="Generate video using ByteDance models via api based on prompt",
|
||||||
|
inputs=[
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"model",
|
||||||
|
options=[model.value for model in Text2VideoModelName],
|
||||||
|
default=Text2VideoModelName.seedance_1_pro.value,
|
||||||
|
tooltip="Model name",
|
||||||
|
),
|
||||||
|
comfy_io.String.Input(
|
||||||
|
"prompt",
|
||||||
|
multiline=True,
|
||||||
|
tooltip="The text prompt used to generate the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"resolution",
|
||||||
|
options=["480p", "720p", "1080p"],
|
||||||
|
tooltip="The resolution of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"aspect_ratio",
|
||||||
|
options=["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||||
|
tooltip="The aspect ratio of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"duration",
|
||||||
|
default=5,
|
||||||
|
min=3,
|
||||||
|
max=12,
|
||||||
|
step=1,
|
||||||
|
tooltip="The duration of the output video in seconds.",
|
||||||
|
display_mode=comfy_io.NumberDisplay.slider,
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"seed",
|
||||||
|
default=0,
|
||||||
|
min=0,
|
||||||
|
max=2147483647,
|
||||||
|
step=1,
|
||||||
|
display_mode=comfy_io.NumberDisplay.number,
|
||||||
|
control_after_generate=True,
|
||||||
|
tooltip="Seed to use for generation.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"camera_fixed",
|
||||||
|
default=False,
|
||||||
|
tooltip="Specifies whether to fix the camera. The platform appends an instruction "
|
||||||
|
"to fix the camera to your prompt, but does not guarantee the actual effect.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"watermark",
|
||||||
|
default=True,
|
||||||
|
tooltip="Whether to add an \"AI generated\" watermark to the video.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
comfy_io.Video.Output(),
|
||||||
|
],
|
||||||
|
hidden=[
|
||||||
|
comfy_io.Hidden.auth_token_comfy_org,
|
||||||
|
comfy_io.Hidden.api_key_comfy_org,
|
||||||
|
comfy_io.Hidden.unique_id,
|
||||||
|
],
|
||||||
|
is_api_node=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def execute(
|
||||||
|
cls,
|
||||||
|
model: str,
|
||||||
|
prompt: str,
|
||||||
|
resolution: str,
|
||||||
|
aspect_ratio: str,
|
||||||
|
duration: int,
|
||||||
|
seed: int,
|
||||||
|
camera_fixed: bool,
|
||||||
|
watermark: bool,
|
||||||
|
) -> comfy_io.NodeOutput:
|
||||||
|
validate_string(prompt, strip_whitespace=True, min_length=1)
|
||||||
|
raise_if_text_params(prompt, ["resolution", "ratio", "duration", "seed", "camerafixed", "watermark"])
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
f"{prompt} "
|
||||||
|
f"--resolution {resolution} "
|
||||||
|
f"--ratio {aspect_ratio} "
|
||||||
|
f"--duration {duration} "
|
||||||
|
f"--seed {seed} "
|
||||||
|
f"--camerafixed {str(camera_fixed).lower()} "
|
||||||
|
f"--watermark {str(watermark).lower()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
auth_kwargs = {
|
||||||
|
"auth_token": cls.hidden.auth_token_comfy_org,
|
||||||
|
"comfy_api_key": cls.hidden.api_key_comfy_org,
|
||||||
|
}
|
||||||
|
return await process_video_task(
|
||||||
|
request_model=Text2VideoTaskCreationRequest,
|
||||||
|
payload=Text2VideoTaskCreationRequest(
|
||||||
|
model=model,
|
||||||
|
content=[TaskTextContent(text=prompt)],
|
||||||
|
),
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
node_id=cls.hidden.unique_id,
|
||||||
|
estimated_duration=max(1, math.ceil(VIDEO_TASKS_EXECUTION_TIME[model][resolution] * (duration / 10.0))),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ByteDanceImageToVideoNode(comfy_io.ComfyNode):
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return comfy_io.Schema(
|
||||||
|
node_id="ByteDanceImageToVideoNode",
|
||||||
|
display_name="ByteDance Image to Video",
|
||||||
|
category="api node/video/ByteDance",
|
||||||
|
description="Generate video using ByteDance models via api based on image and prompt",
|
||||||
|
inputs=[
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"model",
|
||||||
|
options=[model.value for model in Image2VideoModelName],
|
||||||
|
default=Image2VideoModelName.seedance_1_pro.value,
|
||||||
|
tooltip="Model name",
|
||||||
|
),
|
||||||
|
comfy_io.String.Input(
|
||||||
|
"prompt",
|
||||||
|
multiline=True,
|
||||||
|
tooltip="The text prompt used to generate the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Image.Input(
|
||||||
|
"image",
|
||||||
|
tooltip="First frame to be used for the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"resolution",
|
||||||
|
options=["480p", "720p", "1080p"],
|
||||||
|
tooltip="The resolution of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"aspect_ratio",
|
||||||
|
options=["adaptive", "16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||||
|
tooltip="The aspect ratio of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"duration",
|
||||||
|
default=5,
|
||||||
|
min=3,
|
||||||
|
max=12,
|
||||||
|
step=1,
|
||||||
|
tooltip="The duration of the output video in seconds.",
|
||||||
|
display_mode=comfy_io.NumberDisplay.slider,
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"seed",
|
||||||
|
default=0,
|
||||||
|
min=0,
|
||||||
|
max=2147483647,
|
||||||
|
step=1,
|
||||||
|
display_mode=comfy_io.NumberDisplay.number,
|
||||||
|
control_after_generate=True,
|
||||||
|
tooltip="Seed to use for generation.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"camera_fixed",
|
||||||
|
default=False,
|
||||||
|
tooltip="Specifies whether to fix the camera. The platform appends an instruction "
|
||||||
|
"to fix the camera to your prompt, but does not guarantee the actual effect.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"watermark",
|
||||||
|
default=True,
|
||||||
|
tooltip="Whether to add an \"AI generated\" watermark to the video.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
comfy_io.Video.Output(),
|
||||||
|
],
|
||||||
|
hidden=[
|
||||||
|
comfy_io.Hidden.auth_token_comfy_org,
|
||||||
|
comfy_io.Hidden.api_key_comfy_org,
|
||||||
|
comfy_io.Hidden.unique_id,
|
||||||
|
],
|
||||||
|
is_api_node=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def execute(
|
||||||
|
cls,
|
||||||
|
model: str,
|
||||||
|
prompt: str,
|
||||||
|
image: torch.Tensor,
|
||||||
|
resolution: str,
|
||||||
|
aspect_ratio: str,
|
||||||
|
duration: int,
|
||||||
|
seed: int,
|
||||||
|
camera_fixed: bool,
|
||||||
|
watermark: bool,
|
||||||
|
) -> comfy_io.NodeOutput:
|
||||||
|
validate_string(prompt, strip_whitespace=True, min_length=1)
|
||||||
|
raise_if_text_params(prompt, ["resolution", "ratio", "duration", "seed", "camerafixed", "watermark"])
|
||||||
|
validate_image_dimensions(image, min_width=300, min_height=300, max_width=6000, max_height=6000)
|
||||||
|
validate_image_aspect_ratio_range(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5
|
||||||
|
|
||||||
|
auth_kwargs = {
|
||||||
|
"auth_token": cls.hidden.auth_token_comfy_org,
|
||||||
|
"comfy_api_key": cls.hidden.api_key_comfy_org,
|
||||||
|
}
|
||||||
|
|
||||||
|
image_url = (await upload_images_to_comfyapi(image, max_images=1, auth_kwargs=auth_kwargs))[0]
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
f"{prompt} "
|
||||||
|
f"--resolution {resolution} "
|
||||||
|
f"--ratio {aspect_ratio} "
|
||||||
|
f"--duration {duration} "
|
||||||
|
f"--seed {seed} "
|
||||||
|
f"--camerafixed {str(camera_fixed).lower()} "
|
||||||
|
f"--watermark {str(watermark).lower()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return await process_video_task(
|
||||||
|
request_model=Image2VideoTaskCreationRequest,
|
||||||
|
payload=Image2VideoTaskCreationRequest(
|
||||||
|
model=model,
|
||||||
|
content=[TaskTextContent(text=prompt), TaskImageContent(image_url=TaskImageContentUrl(url=image_url))],
|
||||||
|
),
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
node_id=cls.hidden.unique_id,
|
||||||
|
estimated_duration=max(1, math.ceil(VIDEO_TASKS_EXECUTION_TIME[model][resolution] * (duration / 10.0))),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ByteDanceFirstLastFrameNode(comfy_io.ComfyNode):
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return comfy_io.Schema(
|
||||||
|
node_id="ByteDanceFirstLastFrameNode",
|
||||||
|
display_name="ByteDance First-Last-Frame to Video",
|
||||||
|
category="api node/video/ByteDance",
|
||||||
|
description="Generate video using prompt and first and last frames.",
|
||||||
|
inputs=[
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"model",
|
||||||
|
options=[Image2VideoModelName.seedance_1_lite.value],
|
||||||
|
default=Image2VideoModelName.seedance_1_lite.value,
|
||||||
|
tooltip="Model name",
|
||||||
|
),
|
||||||
|
comfy_io.String.Input(
|
||||||
|
"prompt",
|
||||||
|
multiline=True,
|
||||||
|
tooltip="The text prompt used to generate the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Image.Input(
|
||||||
|
"first_frame",
|
||||||
|
tooltip="First frame to be used for the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Image.Input(
|
||||||
|
"last_frame",
|
||||||
|
tooltip="Last frame to be used for the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"resolution",
|
||||||
|
options=["480p", "720p", "1080p"],
|
||||||
|
tooltip="The resolution of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"aspect_ratio",
|
||||||
|
options=["adaptive", "16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||||
|
tooltip="The aspect ratio of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"duration",
|
||||||
|
default=5,
|
||||||
|
min=3,
|
||||||
|
max=12,
|
||||||
|
step=1,
|
||||||
|
tooltip="The duration of the output video in seconds.",
|
||||||
|
display_mode=comfy_io.NumberDisplay.slider,
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"seed",
|
||||||
|
default=0,
|
||||||
|
min=0,
|
||||||
|
max=2147483647,
|
||||||
|
step=1,
|
||||||
|
display_mode=comfy_io.NumberDisplay.number,
|
||||||
|
control_after_generate=True,
|
||||||
|
tooltip="Seed to use for generation.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"camera_fixed",
|
||||||
|
default=False,
|
||||||
|
tooltip="Specifies whether to fix the camera. The platform appends an instruction "
|
||||||
|
"to fix the camera to your prompt, but does not guarantee the actual effect.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"watermark",
|
||||||
|
default=True,
|
||||||
|
tooltip="Whether to add an \"AI generated\" watermark to the video.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
comfy_io.Video.Output(),
|
||||||
|
],
|
||||||
|
hidden=[
|
||||||
|
comfy_io.Hidden.auth_token_comfy_org,
|
||||||
|
comfy_io.Hidden.api_key_comfy_org,
|
||||||
|
comfy_io.Hidden.unique_id,
|
||||||
|
],
|
||||||
|
is_api_node=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def execute(
|
||||||
|
cls,
|
||||||
|
model: str,
|
||||||
|
prompt: str,
|
||||||
|
first_frame: torch.Tensor,
|
||||||
|
last_frame: torch.Tensor,
|
||||||
|
resolution: str,
|
||||||
|
aspect_ratio: str,
|
||||||
|
duration: int,
|
||||||
|
seed: int,
|
||||||
|
camera_fixed: bool,
|
||||||
|
watermark: bool,
|
||||||
|
) -> comfy_io.NodeOutput:
|
||||||
|
validate_string(prompt, strip_whitespace=True, min_length=1)
|
||||||
|
raise_if_text_params(prompt, ["resolution", "ratio", "duration", "seed", "camerafixed", "watermark"])
|
||||||
|
for i in (first_frame, last_frame):
|
||||||
|
validate_image_dimensions(i, min_width=300, min_height=300, max_width=6000, max_height=6000)
|
||||||
|
validate_image_aspect_ratio_range(i, (2, 5), (5, 2), strict=False) # 0.4 to 2.5
|
||||||
|
|
||||||
|
auth_kwargs = {
|
||||||
|
"auth_token": cls.hidden.auth_token_comfy_org,
|
||||||
|
"comfy_api_key": cls.hidden.api_key_comfy_org,
|
||||||
|
}
|
||||||
|
|
||||||
|
download_urls = await upload_images_to_comfyapi(
|
||||||
|
image_tensor_pair_to_batch(first_frame, last_frame),
|
||||||
|
max_images=2,
|
||||||
|
mime_type="image/png",
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
f"{prompt} "
|
||||||
|
f"--resolution {resolution} "
|
||||||
|
f"--ratio {aspect_ratio} "
|
||||||
|
f"--duration {duration} "
|
||||||
|
f"--seed {seed} "
|
||||||
|
f"--camerafixed {str(camera_fixed).lower()} "
|
||||||
|
f"--watermark {str(watermark).lower()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return await process_video_task(
|
||||||
|
request_model=Image2VideoTaskCreationRequest,
|
||||||
|
payload=Image2VideoTaskCreationRequest(
|
||||||
|
model=model,
|
||||||
|
content=[
|
||||||
|
TaskTextContent(text=prompt),
|
||||||
|
TaskImageContent(image_url=TaskImageContentUrl(url=str(download_urls[0])), role="first_frame"),
|
||||||
|
TaskImageContent(image_url=TaskImageContentUrl(url=str(download_urls[1])), role="last_frame"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
node_id=cls.hidden.unique_id,
|
||||||
|
estimated_duration=max(1, math.ceil(VIDEO_TASKS_EXECUTION_TIME[model][resolution] * (duration / 10.0))),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ByteDanceImageReferenceNode(comfy_io.ComfyNode):
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return comfy_io.Schema(
|
||||||
|
node_id="ByteDanceImageReferenceNode",
|
||||||
|
display_name="ByteDance Reference Images to Video",
|
||||||
|
category="api node/video/ByteDance",
|
||||||
|
description="Generate video using prompt and reference images.",
|
||||||
|
inputs=[
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"model",
|
||||||
|
options=[Image2VideoModelName.seedance_1_lite.value],
|
||||||
|
default=Image2VideoModelName.seedance_1_lite.value,
|
||||||
|
tooltip="Model name",
|
||||||
|
),
|
||||||
|
comfy_io.String.Input(
|
||||||
|
"prompt",
|
||||||
|
multiline=True,
|
||||||
|
tooltip="The text prompt used to generate the video.",
|
||||||
|
),
|
||||||
|
comfy_io.Image.Input(
|
||||||
|
"images",
|
||||||
|
tooltip="One to four images.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"resolution",
|
||||||
|
options=["480p", "720p"],
|
||||||
|
tooltip="The resolution of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Combo.Input(
|
||||||
|
"aspect_ratio",
|
||||||
|
options=["adaptive", "16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||||
|
tooltip="The aspect ratio of the output video.",
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"duration",
|
||||||
|
default=5,
|
||||||
|
min=3,
|
||||||
|
max=12,
|
||||||
|
step=1,
|
||||||
|
tooltip="The duration of the output video in seconds.",
|
||||||
|
display_mode=comfy_io.NumberDisplay.slider,
|
||||||
|
),
|
||||||
|
comfy_io.Int.Input(
|
||||||
|
"seed",
|
||||||
|
default=0,
|
||||||
|
min=0,
|
||||||
|
max=2147483647,
|
||||||
|
step=1,
|
||||||
|
display_mode=comfy_io.NumberDisplay.number,
|
||||||
|
control_after_generate=True,
|
||||||
|
tooltip="Seed to use for generation.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
comfy_io.Boolean.Input(
|
||||||
|
"watermark",
|
||||||
|
default=True,
|
||||||
|
tooltip="Whether to add an \"AI generated\" watermark to the video.",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
comfy_io.Video.Output(),
|
||||||
|
],
|
||||||
|
hidden=[
|
||||||
|
comfy_io.Hidden.auth_token_comfy_org,
|
||||||
|
comfy_io.Hidden.api_key_comfy_org,
|
||||||
|
comfy_io.Hidden.unique_id,
|
||||||
|
],
|
||||||
|
is_api_node=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def execute(
|
||||||
|
cls,
|
||||||
|
model: str,
|
||||||
|
prompt: str,
|
||||||
|
images: torch.Tensor,
|
||||||
|
resolution: str,
|
||||||
|
aspect_ratio: str,
|
||||||
|
duration: int,
|
||||||
|
seed: int,
|
||||||
|
watermark: bool,
|
||||||
|
) -> comfy_io.NodeOutput:
|
||||||
|
validate_string(prompt, strip_whitespace=True, min_length=1)
|
||||||
|
raise_if_text_params(prompt, ["resolution", "ratio", "duration", "seed", "watermark"])
|
||||||
|
for image in images:
|
||||||
|
validate_image_dimensions(image, min_width=300, min_height=300, max_width=6000, max_height=6000)
|
||||||
|
validate_image_aspect_ratio_range(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5
|
||||||
|
|
||||||
|
auth_kwargs = {
|
||||||
|
"auth_token": cls.hidden.auth_token_comfy_org,
|
||||||
|
"comfy_api_key": cls.hidden.api_key_comfy_org,
|
||||||
|
}
|
||||||
|
|
||||||
|
image_urls = await upload_images_to_comfyapi(
|
||||||
|
images, max_images=4, mime_type="image/png", auth_kwargs=auth_kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
f"{prompt} "
|
||||||
|
f"--resolution {resolution} "
|
||||||
|
f"--ratio {aspect_ratio} "
|
||||||
|
f"--duration {duration} "
|
||||||
|
f"--seed {seed} "
|
||||||
|
f"--watermark {str(watermark).lower()}"
|
||||||
|
)
|
||||||
|
x = [
|
||||||
|
TaskTextContent(text=prompt),
|
||||||
|
*[TaskImageContent(image_url=TaskImageContentUrl(url=str(i)), role="reference_image") for i in image_urls]
|
||||||
|
]
|
||||||
|
return await process_video_task(
|
||||||
|
request_model=Image2VideoTaskCreationRequest,
|
||||||
|
payload=Image2VideoTaskCreationRequest(
|
||||||
|
model=model,
|
||||||
|
content=x,
|
||||||
|
),
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
node_id=cls.hidden.unique_id,
|
||||||
|
estimated_duration=max(1, math.ceil(VIDEO_TASKS_EXECUTION_TIME[model][resolution] * (duration / 10.0))),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def process_video_task(
|
||||||
|
request_model: Type[T],
|
||||||
|
payload: Union[Text2VideoTaskCreationRequest, Image2VideoTaskCreationRequest],
|
||||||
|
auth_kwargs: dict,
|
||||||
|
node_id: str,
|
||||||
|
estimated_duration: int | None,
|
||||||
|
) -> comfy_io.NodeOutput:
|
||||||
|
initial_response = await SynchronousOperation(
|
||||||
|
endpoint=ApiEndpoint(
|
||||||
|
path=BYTEPLUS_TASK_ENDPOINT,
|
||||||
|
method=HttpMethod.POST,
|
||||||
|
request_model=request_model,
|
||||||
|
response_model=TaskCreationResponse,
|
||||||
|
),
|
||||||
|
request=payload,
|
||||||
|
auth_kwargs=auth_kwargs,
|
||||||
|
).execute()
|
||||||
|
response = await poll_until_finished(
|
||||||
|
auth_kwargs,
|
||||||
|
initial_response.id,
|
||||||
|
estimated_duration=estimated_duration,
|
||||||
|
node_id=node_id,
|
||||||
|
)
|
||||||
|
return comfy_io.NodeOutput(await download_url_to_video_output(get_video_url_from_task_status(response)))
|
||||||
|
|
||||||
|
|
||||||
|
def raise_if_text_params(prompt: str, text_params: list[str]) -> None:
|
||||||
|
for i in text_params:
|
||||||
|
if f"--{i} " in prompt:
|
||||||
|
raise ValueError(
|
||||||
|
f"--{i} is not allowed in the prompt, use the appropriated widget input to change this value."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ByteDanceExtension(ComfyExtension):
|
class ByteDanceExtension(ComfyExtension):
|
||||||
@override
|
@override
|
||||||
async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
|
async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
|
||||||
return [
|
return [
|
||||||
ByteDanceImageNode,
|
ByteDanceImageNode,
|
||||||
ByteDanceImageEditNode,
|
ByteDanceImageEditNode,
|
||||||
|
ByteDanceTextToVideoNode,
|
||||||
|
ByteDanceImageToVideoNode,
|
||||||
|
ByteDanceFirstLastFrameNode,
|
||||||
|
ByteDanceImageReferenceNode,
|
||||||
]
|
]
|
||||||
|
|
||||||
async def comfy_entrypoint() -> ByteDanceExtension:
|
async def comfy_entrypoint() -> ByteDanceExtension:
|
||||||
|
Reference in New Issue
Block a user