Skip to content

Fix/wan animate frame alignment #1335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zwplus wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Fix/wan animate frame alignment #1335

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0c09523
Fix wan-animate temporal alignment without dropping frames
zwplus Mar 7, 2026
9e636c8
Set wan-animate examples to 77 frames
zwplus Mar 7, 2026
48a6826
Revert "Set wan-animate examples to 77 frames"
zwplus Mar 7, 2026
42c6903
Use 77 frames in wan-animate examples
zwplus Mar 7, 2026
1304e6f
Update examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py
zwplus Mar 7, 2026
91ae54f
Update examples/wanvideo/model_training/validate_full/Wan2.2-Animate-…
zwplus Mar 7, 2026
d5b7ad7
Update examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-…
zwplus Mar 7, 2026
99b1887
Remove UTF-8 BOM from wan-animate examples
zwplus Mar 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 12 additions & 26 deletions diffsynth/pipelines/wan_video.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16):
WanVideoUnit_FunCameraControl(),
WanVideoUnit_SpeedControl(),
WanVideoUnit_VACE(),
WanVideoUnit_AnimateVideoSplit(),
WanVideoUnit_AnimatePoseLatents(),
WanVideoUnit_AnimateFacePixelValues(),
WanVideoUnit_AnimateInpaint(),
Expand Down Expand Up @@ -351,12 +350,15 @@ def process(self, pipe: WanVideoPipeline, height, width, num_frames):
class WanVideoUnit_NoiseInitializer(PipelineUnit):
def __init__(self):
super().__init__(
input_params=("height", "width", "num_frames", "seed", "rand_device", "vace_reference_image"),
input_params=("height", "width", "num_frames", "seed", "rand_device", "vace_reference_image", "input_image", "animate_pose_video"),
output_params=("noise",)
)

def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_device, vace_reference_image):
def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_device, vace_reference_image, input_image, animate_pose_video):
length = (num_frames - 1) // 4 + 1
# For wan-animate, input_image is a single reference frame; align time dimension.
if input_image is not None and animate_pose_video is not None:
length += 1
if vace_reference_image is not None:
f = len(vace_reference_image) if isinstance(vace_reference_image, list) else 1
length += f
Expand All @@ -371,12 +373,12 @@ def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_
class WanVideoUnit_InputVideoEmbedder(PipelineUnit):
def __init__(self):
super().__init__(
input_params=("input_video", "noise", "tiled", "tile_size", "tile_stride", "vace_reference_image"),
input_params=("input_video", "noise", "tiled", "tile_size", "tile_stride", "vace_reference_image", "input_image", "animate_pose_video"),
output_params=("latents", "input_latents"),
onload_model_names=("vae",)
)

def process(self, pipe: WanVideoPipeline, input_video, noise, tiled, tile_size, tile_stride, vace_reference_image):
def process(self, pipe: WanVideoPipeline, input_video, noise, tiled, tile_size, tile_stride, vace_reference_image, input_image, animate_pose_video):
if input_video is None:
return {"latents": noise}
pipe.load_models_to_device(self.onload_model_names)
Expand All @@ -388,6 +390,11 @@ def process(self, pipe: WanVideoPipeline, input_video, noise, tiled, tile_size,
vace_reference_image = pipe.preprocess_video(vace_reference_image)
vace_reference_latents = pipe.vae.encode(vace_reference_image, device=pipe.device).to(dtype=pipe.torch_dtype, device=pipe.device)
input_latents = torch.concat([vace_reference_latents, input_latents], dim=2)
# For wan-animate, prepend the single reference frame latent
if input_image is not None and animate_pose_video is not None:
input_image = pipe.preprocess_video([input_image])
input_image_latents = pipe.vae.encode(input_image, device=pipe.device).to(dtype=pipe.torch_dtype, device=pipe.device)
input_latents = torch.concat([input_image_latents, input_latents], dim=2)
if pipe.scheduler.training:
return {"latents": noise, "input_latents": input_latents}
else:
Expand Down Expand Up @@ -903,27 +910,6 @@ def process(self, pipe: WanVideoPipeline, latents, motion_latents, drop_motion_f
return {"latents": latents}


class WanVideoUnit_AnimateVideoSplit(PipelineUnit):
def __init__(self):
super().__init__(
input_params=("input_video", "animate_pose_video", "animate_face_video", "animate_inpaint_video", "animate_mask_video"),
output_params=("animate_pose_video", "animate_face_video", "animate_inpaint_video", "animate_mask_video")
)

def process(self, pipe: WanVideoPipeline, input_video, animate_pose_video, animate_face_video, animate_inpaint_video, animate_mask_video):
if input_video is None:
return {}
if animate_pose_video is not None:
animate_pose_video = animate_pose_video[:len(input_video) - 4]
if animate_face_video is not None:
animate_face_video = animate_face_video[:len(input_video) - 4]
if animate_inpaint_video is not None:
animate_inpaint_video = animate_inpaint_video[:len(input_video) - 4]
if animate_mask_video is not None:
animate_mask_video = animate_mask_video[:len(input_video) - 4]
return {"animate_pose_video": animate_pose_video, "animate_face_video": animate_face_video, "animate_inpaint_video": animate_inpaint_video, "animate_mask_video": animate_mask_video}


class WanVideoUnit_AnimatePoseLatents(PipelineUnit):
def __init__(self):
super().__init__(
Expand Down
19 changes: 10 additions & 9 deletions examples/wanvideo/model_inference/Wan2.2-Animate-14B.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import torch
import torch
Comment thread
zwplus marked this conversation as resolved.
Outdated
from PIL import Image
from diffsynth.core import load_state_dict
from diffsynth.utils.data import save_video, VideoData
Expand Down Expand Up @@ -26,15 +26,15 @@

# Animate
input_image = Image.open("data/examples/wan/animate/animate_input_image.png")
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4").raw_data()[:81-4]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4").raw_data()[:81-4]
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4").raw_data()[:77]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4").raw_data()[:77]
video = pipe(
prompt="视频中的人在做动作",
seed=0, tiled=True,
input_image=input_image,
animate_pose_video=animate_pose_video,
animate_face_video=animate_face_video,
num_frames=81, height=720, width=1280,
num_frames=77, height=720, width=1280,
Comment on lines +29 to +37
Copy link
Copy Markdown
Contributor

@gemini-code-assist gemini-code-assist Bot Mar 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The number 77 is used in multiple places within this block. To improve readability and maintainability, it's good practice to define it as a variable before this block and reuse it. This avoids 'magic numbers' and makes it easier to change this value in the future. For example:

num_frames_to_process = 77
animate_pose_video = VideoData(...).raw_data()[:num_frames_to_process]
animate_face_video = VideoData(...).raw_data()[:num_frames_to_process]
video = pipe(
    ...
    num_frames=num_frames_to_process,
    ...
)

num_inference_steps=20, cfg_scale=1,
)
save_video(video, "video_1_Wan2.2-Animate-14B.mp4", fps=15, quality=5)
Expand All @@ -44,10 +44,10 @@
lora_state_dict = load_state_dict("models/Wan-AI/Wan2.2-Animate-14B/relighting_lora.ckpt", torch_dtype=torch.bfloat16, device="cuda")["state_dict"]
pipe.load_lora(pipe.dit, state_dict=lora_state_dict)
input_image = Image.open("data/examples/wan/animate/replace_input_image.png")
animate_pose_video = VideoData("data/examples/wan/animate/replace_pose_video.mp4").raw_data()[:81-4]
animate_face_video = VideoData("data/examples/wan/animate/replace_face_video.mp4").raw_data()[:81-4]
animate_inpaint_video = VideoData("data/examples/wan/animate/replace_inpaint_video.mp4").raw_data()[:81-4]
animate_mask_video = VideoData("data/examples/wan/animate/replace_mask_video.mp4").raw_data()[:81-4]
animate_pose_video = VideoData("data/examples/wan/animate/replace_pose_video.mp4").raw_data()[:77]
animate_face_video = VideoData("data/examples/wan/animate/replace_face_video.mp4").raw_data()[:77]
animate_inpaint_video = VideoData("data/examples/wan/animate/replace_inpaint_video.mp4").raw_data()[:77]
animate_mask_video = VideoData("data/examples/wan/animate/replace_mask_video.mp4").raw_data()[:77]
video = pipe(
prompt="视频中的人在做动作",
seed=0, tiled=True,
Expand All @@ -56,7 +56,8 @@
animate_face_video=animate_face_video,
animate_inpaint_video=animate_inpaint_video,
animate_mask_video=animate_mask_video,
num_frames=81, height=720, width=1280,
num_frames=77, height=720, width=1280,
num_inference_steps=20, cfg_scale=1,
)
save_video(video, "video_2_Wan2.2-Animate-14B.mp4", fps=15, quality=5)

19 changes: 10 additions & 9 deletions examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import torch
import torch
Comment thread
zwplus marked this conversation as resolved.
Outdated
from PIL import Image
from diffsynth.core import load_state_dict
from diffsynth.utils.data import save_video, VideoData
Expand Down Expand Up @@ -37,15 +37,15 @@

# Animate
input_image = Image.open("data/examples/wan/animate/animate_input_image.png")
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4").raw_data()[:81-4]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4").raw_data()[:81-4]
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4").raw_data()[:77]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4").raw_data()[:77]
video = pipe(
prompt="视频中的人在做动作",
seed=0, tiled=True,
input_image=input_image,
animate_pose_video=animate_pose_video,
animate_face_video=animate_face_video,
num_frames=81, height=720, width=1280,
num_frames=77, height=720, width=1280,
Comment on lines +40 to +48
Copy link
Copy Markdown
Contributor

@gemini-code-assist gemini-code-assist Bot Mar 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The number 77 is used in multiple places within this block. To improve readability and maintainability, it's good practice to define it as a variable before this block and reuse it. This avoids 'magic numbers' and makes it easier to change this value in the future. For example:

num_frames_to_process = 77
animate_pose_video = VideoData(...).raw_data()[:num_frames_to_process]
animate_face_video = VideoData(...).raw_data()[:num_frames_to_process]
video = pipe(
    ...
    num_frames=num_frames_to_process,
    ...
)

num_inference_steps=20, cfg_scale=1,
)
save_video(video, "video_1_Wan2.2-Animate-14B.mp4", fps=15, quality=5)
Expand All @@ -56,10 +56,10 @@
lora_state_dict = {i: lora_state_dict[i].to(torch.bfloat16) for i in lora_state_dict}
pipe.load_lora(pipe.dit, state_dict=lora_state_dict)
input_image = Image.open("data/examples/wan/animate/replace_input_image.png")
animate_pose_video = VideoData("data/examples/wan/animate/replace_pose_video.mp4").raw_data()[:81-4]
animate_face_video = VideoData("data/examples/wan/animate/replace_face_video.mp4").raw_data()[:81-4]
animate_inpaint_video = VideoData("data/examples/wan/animate/replace_inpaint_video.mp4").raw_data()[:81-4]
animate_mask_video = VideoData("data/examples/wan/animate/replace_mask_video.mp4").raw_data()[:81-4]
animate_pose_video = VideoData("data/examples/wan/animate/replace_pose_video.mp4").raw_data()[:77]
animate_face_video = VideoData("data/examples/wan/animate/replace_face_video.mp4").raw_data()[:77]
animate_inpaint_video = VideoData("data/examples/wan/animate/replace_inpaint_video.mp4").raw_data()[:77]
animate_mask_video = VideoData("data/examples/wan/animate/replace_mask_video.mp4").raw_data()[:77]
video = pipe(
prompt="视频中的人在做动作",
seed=0, tiled=True,
Expand All @@ -68,7 +68,8 @@
animate_face_video=animate_face_video,
animate_inpaint_video=animate_inpaint_video,
animate_mask_video=animate_mask_video,
num_frames=81, height=720, width=1280,
num_frames=77, height=720, width=1280,
num_inference_steps=20, cfg_scale=1,
)
save_video(video, "video_2_Wan2.2-Animate-14B.mp4", fps=15, quality=5)

11 changes: 6 additions & 5 deletions examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import torch
import torch
Comment thread
zwplus marked this conversation as resolved.
Outdated
from PIL import Image
from diffsynth.utils.data import save_video, VideoData
from diffsynth.core import load_state_dict
Expand All @@ -19,15 +19,16 @@
pipe.animate_adapter.load_state_dict(state_dict, strict=False)

input_image = VideoData("data/example_video_dataset/animate/animate_output.mp4", height=480, width=832)[0]
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4", height=480, width=832).raw_data()[:81-4]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4", height=512, width=512).raw_data()[:81-4]
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4", height=480, width=832).raw_data()[:77]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4", height=512, width=512).raw_data()[:77]
video = pipe(
prompt="视频中的人在做动作",
seed=0, tiled=True,
input_image=input_image,
animate_pose_video=animate_pose_video,
animate_face_video=animate_face_video,
num_frames=81, height=480, width=832,
num_frames=77, height=480, width=832,
Comment on lines +22 to +30
Copy link
Copy Markdown
Contributor

@gemini-code-assist gemini-code-assist Bot Mar 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The number 77 is used in multiple places within this block. To improve readability and maintainability, it's good practice to define it as a variable before this block and reuse it. This avoids 'magic numbers' and makes it easier to change this value in the future. For example:

num_frames_to_process = 77
animate_pose_video = VideoData(...).raw_data()[:num_frames_to_process]
animate_face_video = VideoData(...).raw_data()[:num_frames_to_process]
video = pipe(
    ...
    num_frames=num_frames_to_process,
    ...
)

num_inference_steps=20, cfg_scale=1,
)
save_video(video, "video_Wan2.2-Animate-14B.mp4", fps=15, quality=5)
save_video(video, "video_Wan2.2-Animate-14B.mp4", fps=15, quality=5)

11 changes: 6 additions & 5 deletions examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import torch
import torch
Comment thread
zwplus marked this conversation as resolved.
Outdated
from PIL import Image
from diffsynth.utils.data import save_video, VideoData
from diffsynth.core import load_state_dict
Expand All @@ -18,15 +18,16 @@
pipe.load_lora(pipe.dit, "models/train/Wan2.2-Animate-14B_lora/epoch-4.safetensors", alpha=1)

input_image = VideoData("data/example_video_dataset/animate/animate_output.mp4", height=480, width=832)[0]
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4", height=480, width=832).raw_data()[:81-4]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4", height=512, width=512).raw_data()[:81-4]
animate_pose_video = VideoData("data/examples/wan/animate/animate_pose_video.mp4", height=480, width=832).raw_data()[:77]
animate_face_video = VideoData("data/examples/wan/animate/animate_face_video.mp4", height=512, width=512).raw_data()[:77]
video = pipe(
prompt="视频中的人在做动作",
seed=0, tiled=True,
input_image=input_image,
animate_pose_video=animate_pose_video,
animate_face_video=animate_face_video,
num_frames=81, height=480, width=832,
num_frames=77, height=480, width=832,
Comment on lines +21 to +29
Copy link
Copy Markdown
Contributor

@gemini-code-assist gemini-code-assist Bot Mar 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The number 77 is used in multiple places within this block. To improve readability and maintainability, it's good practice to define it as a variable before this block and reuse it. This avoids 'magic numbers' and makes it easier to change this value in the future. For example:

num_frames_to_process = 77
animate_pose_video = VideoData(...).raw_data()[:num_frames_to_process]
animate_face_video = VideoData(...).raw_data()[:num_frames_to_process]
video = pipe(
    ...
    num_frames=num_frames_to_process,
    ...
)

num_inference_steps=20, cfg_scale=1,
)
save_video(video, "video_Wan2.2-Animate-14B.mp4", fps=15, quality=5)
save_video(video, "video_Wan2.2-Animate-14B.mp4", fps=15, quality=5)