Merge 628af864af into 1ac78180b3

Make this deterministic so speeds dont change base of load order. Load
them in reverse order so whatever the caller lists first is the top
priority.

comfy/model_management.py

@@ -721,13 +721,15 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimu
     else:
         minimum_memory_required = max(inference_memory, minimum_memory_required + extra_reserved_memory())
 
-    models_temp = set()
+    # Order-preserving dedup. A plain set() would randomize iteration order across runs
+    models_temp = {}
     for m in models:
-        models_temp.add(m)
+        models_temp[m] = None
         for mm in m.model_patches_models():
-            models_temp.add(mm)
+            models_temp[mm] = None
 
-    models = models_temp
+    models = list(models_temp)
+    models.reverse()
 
     models_to_load = []
 

comfy/model_prefetch.py

@@ -37,7 +37,8 @@ def prefetch_queue_pop(queue, device, module):
     consumed = queue.pop(0)
     if consumed is not None:
         offload_stream, prefetch_state = consumed
-        offload_stream.wait_stream(comfy.model_management.current_stream(device))
+        if offload_stream is not None:
+            offload_stream.wait_stream(comfy.model_management.current_stream(device))
         _, comfy_modules = prefetch_state
         if comfy_modules is not None:
             cleanup_prefetched_modules(comfy_modules)

comfy/sampler_helpers.py

@@ -89,7 +89,8 @@ def get_additional_models(conds, dtype):
         gligen += get_models_from_cond(conds[k], "gligen")
         add_models += get_models_from_cond(conds[k], "additional_models")
 
-    control_nets = set(cnets)
+    # Order-preserving dedup. A plain set() would randomize iteration order across runs
+    control_nets = list(dict.fromkeys(cnets))
 
     inference_memory = 0
     control_models = []

comfy_api/latest/_input_impl/video_types.py

@@ -8,12 +8,15 @@ import av
 import io
 import itertools
 import json
+import logging
 import numpy as np
 import math
 import torch
 from .._util import VideoContainer, VideoCodec, VideoComponents
 import logging
 
+logger = logging.getLogger(__name__)
+
 
 def container_to_output_format(container_format: str | None) -> str | None:
     """
@@ -464,6 +467,16 @@ class VideoFromComponents(VideoInput):
         metadata: Optional[dict] = None,
     ):
         """Save the video to a file path or BytesIO buffer."""
+        def mux_packets(container: av.OutputContainer, packets):
+            if packets is None:
+                return
+            if isinstance(packets, (list, tuple)):
+                for packet in packets:
+                    if packet is not None:
+                        container.mux(packet)
+                return
+            container.mux(packets)
+
         if format != VideoContainer.AUTO and format != VideoContainer.MP4:
             raise ValueError("Only MP4 format is supported for now")
         if codec != VideoCodec.AUTO and codec != VideoCodec.H264:
@@ -495,6 +508,8 @@ class VideoFromComponents(VideoInput):
                 audio_sample_rate = int(self.__components.audio['sample_rate'])
                 waveform = self.__components.audio['waveform']
                 waveform = waveform[0, :, :math.ceil((audio_sample_rate / frame_rate) * self.__components.images.shape[0])]
+                # Guard ffmpeg encoder against invalid upstream audio (NaN/Inf/out-of-range).
+                waveform = torch.nan_to_num(waveform, nan=0.0, posinf=0.0, neginf=0.0).clamp(-1.0, 1.0)
                 layout = {1: 'mono', 2: 'stereo', 6: '5.1'}.get(waveform.shape[0], 'stereo')
                 audio_stream = output.add_stream('aac', rate=audio_sample_rate, layout=layout)
 
@@ -511,13 +526,26 @@ class VideoFromComponents(VideoInput):
             output.mux(packet)
 
             if audio_stream and self.__components.audio:
-                frame = av.AudioFrame.from_ndarray(waveform.float().cpu().contiguous().numpy(), format='fltp', layout=layout)
-                frame.sample_rate = audio_sample_rate
-                frame.pts = 0
-                output.mux(audio_stream.encode(frame))
+                encoded_audio_packets = None
+                flush_audio_packets = None
+                try:
+                    audio_np = waveform.float().cpu().contiguous().numpy()
+                    if not np.isfinite(audio_np).all():
+                        audio_np = np.nan_to_num(audio_np, nan=0.0, posinf=0.0, neginf=0.0)
 
-                # Flush encoder
-                output.mux(audio_stream.encode(None))
+                    frame = av.AudioFrame.from_ndarray(audio_np, format='fltp', layout=layout)
+                    frame.sample_rate = audio_sample_rate
+                    frame.pts = 0
+                    encoded_audio_packets = audio_stream.encode(frame)
+                    flush_audio_packets = audio_stream.encode(None)
+                except (av.error.ArgumentError, ValueError, TypeError) as exc:
+                    logger.error(
+                        "Audio encode failed due to invalid audio data; skipping audio track and saving video-only output: %s",
+                        exc,
+                    )
+                else:
+                    mux_packets(output, encoded_audio_packets)
+                    mux_packets(output, flush_audio_packets)
 
     def as_trimmed(
         self,

comfy_extras/frame_interpolation_models/film_net.py

@@ -199,6 +199,9 @@ class FILMNet(nn.Module):
     def get_dtype(self):
         return self.extract.extract_sublevels.convs[0][0].conv.weight.dtype
 
+    def memory_used_forward(self, shape, dtype):
+        return 1700 * shape[1] * shape[2] * dtype.itemsize
+
     def _build_warp_grids(self, H, W, device):
         """Pre-compute warp grids for all pyramid levels."""
         if (H, W) in self._warp_grids:

comfy_extras/frame_interpolation_models/ifnet.py

@@ -74,6 +74,9 @@ class IFNet(nn.Module):
     def get_dtype(self):
         return self.encode.cnn0.weight.dtype
 
+    def memory_used_forward(self, shape, dtype):
+        return 300 * shape[1] * shape[2] * dtype.itemsize
+
     def _build_warp_grids(self, H, W, device):
         if (H, W) in self._warp_grids:
             return

comfy_extras/nodes_frame_interpolation.py

@@ -37,7 +37,7 @@ class FrameInterpolationModelLoader(io.ComfyNode):
         model = cls._detect_and_load(sd)
         dtype = torch.float16 if model_management.should_use_fp16(model_management.get_torch_device()) else torch.float32
         model.eval().to(dtype)
-        patcher = comfy.model_patcher.ModelPatcher(
+        patcher = comfy.model_patcher.CoreModelPatcher(
             model,
             load_device=model_management.get_torch_device(),
             offload_device=model_management.unet_offload_device(),
@@ -98,16 +98,13 @@ class FrameInterpolate(io.ComfyNode):
         if num_frames < 2 or multiplier < 2:
             return io.NodeOutput(images)
 
-        model_management.load_model_gpu(interp_model)
         device = interp_model.load_device
         dtype = interp_model.model_dtype()
         inference_model = interp_model.model
-
-        # Free VRAM for inference activations (model weights + ~20x a single frame's worth)
-        H, W = images.shape[1], images.shape[2]
-        activation_mem = H * W * 3 * images.element_size() * 20
-        model_management.free_memory(activation_mem, device)
+        activation_mem = inference_model.memory_used_forward(images.shape, dtype)
+        model_management.load_models_gpu([interp_model], memory_required=activation_mem)
         align = getattr(inference_model, "pad_align", 1)
+        H, W = images.shape[1], images.shape[2]
 
         # Prepare a single padded frame on device for determining output dimensions
         def prepare_frame(idx):

extra_model_paths.yaml.example

@@ -28,7 +28,7 @@
 #config for a1111 ui
 #all you have to do is uncomment this (remove the #) and change the base_path to where yours is installed
 
-#a111:
+#a1111:
 #     base_path: path/to/stable-diffusion-webui/
 #     checkpoints: models/Stable-diffusion
 #     configs: models/Stable-diffusion