Merge 66bbdefcc0 into a3020f107e

* fix(Video): don't crash on videos with undecodable audio streams

Signed-off-by: bigcat88 <bigcat88@icloud.com>

* Update comfy_api_nodes/util/upload_helpers.py

---------

Signed-off-by: bigcat88 <bigcat88@icloud.com>
Co-authored-by: Alexis Rolland <alexisrolland@hotmail.com>

.github/workflows/cla.yml

@@ -0,0 +1,91 @@
+name: CLA Assistant
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened, synchronize, closed]
+
+permissions:
+  actions: write
+  contents: read # 'read' is enough because signatures live in a REMOTE repo
+  pull-requests: write
+  statuses: write
+
+jobs:
+  cla-assistant:
+    runs-on: ubuntu-latest
+    steps:
+      # The CLA action normally requires every commit author in a PR to sign.
+      # We only want the PR author to sign, so we allowlist all other committers
+      # by computing them from the PR's commits and excluding the PR author.
+      - name: Build author-only allowlist
+        id: allowlist
+        if: >
+          github.event_name == 'pull_request_target' ||
+          (github.event_name == 'issue_comment' && github.event.issue.pull_request && (
+            github.event.comment.body == 'recheck' ||
+            github.event.comment.body == 'I have read and agree to the Contributor License Agreement'
+          ))
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login || github.event.issue.user.login }}
+          BASE_ALLOWLIST: action@github.com,actions-user,ampagent,claude,comfy-pr-bot,GitHub Action,github-actions,github-actions[bot],Glary Bot,Glary-Bot,*[bot]
+        run: |
+          others=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}/commits" --paginate \
+            --jq '.[] | (.author.login // empty), (.committer.login // empty)' \
+            | sort -u | grep -vix "${PR_AUTHOR}" | paste -sd, -)
+          if [ -n "$others" ]; then
+            echo "allowlist=${BASE_ALLOWLIST},${others}" >> "$GITHUB_OUTPUT"
+          else
+            echo "allowlist=${BASE_ALLOWLIST}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: CLA Assistant
+        # Run on PR events, on "recheck" comment, or when someone posts the exact signing phrase.
+        # IMPORTANT: this phrase must match `custom-pr-sign-comment` below.
+        if: >
+          github.event_name == 'pull_request_target' ||
+          (github.event_name == 'issue_comment' && github.event.issue.pull_request && (
+            github.event.comment.body == 'recheck' ||
+            github.event.comment.body == 'I have read and agree to the Contributor License Agreement'
+          ))
+        uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # PAT required to write to the centralized signatures repo.
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        with:
+          # Where the CLA document lives (shown to contributors)
+          path-to-document: https://github.com/Comfy-Org/comfy-cla/blob/main/comfyui_icla.md
+
+          # Centralized signature storage
+          remote-organization-name: comfy-org
+          remote-repository-name: comfy-cla
+          path-to-signatures: signatures/cla.json
+          branch: main
+
+          # Only the PR author must sign: bots plus every non-author committer
+          # are allowlisted via the "Build author-only allowlist" step above.
+          # *[bot] is a catch-all for any GitHub App bot account.
+          allowlist: ${{ steps.allowlist.outputs.allowlist }}
+
+          # Custom PR comment messages
+          custom-notsigned-prcomment: |
+            🎉 Thank you for your contribution, we really appreciate it! 🎉
+
+            Like many open source projects, we require contributors to sign our [Contributor License Agreement (CLA)](https://github.com/Comfy-Org/comfy-cla/blob/main/comfyui_icla.md). A CLA makes the ownership of contributions explicit, so contributors and the project share a clear understanding of how the code can be used. By signing, you:
+
+            - Confirm that you own your contribution.
+            - Keep the right to reuse your own code.
+            - Grant us a copyright license to include and share it within our projects.
+
+            CLAs are standard practice across major open source projects including those under the Apache Software Foundation and the Linux Foundation. Ours is based on the Apache Software Foundation's CLA. Most importantly, it would enable us to relicense the project under a more permissive license in the future, giving the project and its community greater flexibility.
+
+            ✍ **To sign, please post a new comment on this PR with exactly the following text:** ✍
+
+          custom-pr-sign-comment: I have read and agree to the Contributor License Agreement
+
+          custom-allsigned-prcomment: |
+            ✅ All contributors have signed the CLA. Thank you! This PR is ready to be merged.

CLAUDE.md

@@ -1 +0,0 @@
-AGENTS.md

comfy/k_diffusion/sampling.py

@@ -1521,70 +1521,132 @@ def sample_gradient_estimation_cfg_pp(model, x, sigmas, extra_args=None, callbac
     return sample_gradient_estimation(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, ge_gamma=ge_gamma, cfg_pp=True)
 
 
+# Extended Reverse-Time SDE solver (VP ER-SDE-Solver-3). arXiv: https://arxiv.org/abs/2309.06169.
+# Code reference for initial implementation: https://github.com/QinpengCui/ER-SDE-Solver/blob/main/er_sde_solver.py.
 @torch.no_grad()
-def sample_er_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, s_noise=1.0, noise_sampler=None, noise_scaler=None, max_stage=3):
-    """Extended Reverse-Time SDE solver (VP ER-SDE-Solver-3). arXiv: https://arxiv.org/abs/2309.06169.
-    Code reference: https://github.com/QinpengCui/ER-SDE-Solver/blob/main/er_sde_solver.py.
-    """
+def sample_er_sde(
+    model,
+    x: torch.Tensor,
+    sigmas: torch.Tensor,
+    extra_args=None,
+    callback=None,
+    disable=None,
+    eta: float = 1.0,
+    s_noise: float = 1.0,
+    noise_sampler=None,
+    noise_scaler=None,
+    max_stage: int = 3,
+    num_integration_points: int = 200,
+    scaling_power: float = 0.3,
+    scaling_constant: float = 10.0,
+    interpolation_function=torch.lerp,
+    # One of default, ersde or sde.
+    solver_type: str = "default",
+) -> torch.Tensor:
+    model_sampling = model.inner_model.model_patcher.get_model_object("model_sampling")
     extra_args = {} if extra_args is None else extra_args
     seed = extra_args.get("seed", None)
-    noise_sampler = default_noise_sampler(x, seed=seed) if noise_sampler is None else noise_sampler
-    s_noise = s_noise * getattr(model.inner_model.model_patcher.get_model_object('model_sampling'), "noise_scale", 1.0)
-    s_in = x.new_ones([x.shape[0]])
+    eta = max(0.0, eta)
+    if eta > 0:
+        s_noise = s_noise * getattr(model_sampling, "noise_scale", 1.0)
+        if noise_sampler is None:
+            noise_sampler = default_noise_sampler(x, seed=seed)
 
-    def default_er_sde_noise_scaler(x):
-        return x * ((x ** 0.3).exp() + 10.0)
+    s_in = x.new_ones(x.shape[:1])
 
-    noise_scaler = default_er_sde_noise_scaler if noise_scaler is None else noise_scaler
-    num_integration_points = 200.0
-    point_indice = torch.arange(0, num_integration_points, dtype=torch.float32, device=x.device)
+    if solver_type not in {"default", "sde", "ersde"}:
+        raise ValueError("Bad solver_type, must be one of default, ersde or sde")
+    if noise_scaler is None:
+        if solver_type == "sde":
+
+            def noise_scaler(val_x: torch.Tensor) -> torch.Tensor:
+                return val_x ** (eta + 1)
+
+        else:  # default or ersde.
+            solver_type = "ersde"
+
+            def noise_scaler(val_x: torch.Tensor) -> torch.Tensor:
+                rho_sde = val_x * ((val_x**scaling_power).exp_() + scaling_constant)
+                squared_scale = (1.0 - eta**2) * (val_x**2) + (eta**2) * (rho_sde**2)
+                return squared_scale.clamp_min_(1e-09).sqrt_()
+
+    elif solver_type == "default":
+        solver_type = "sde"
+
+    point_indice = torch.arange(
+        0, num_integration_points, dtype=x.dtype, device=x.device
+    )
 
-    model_sampling = model.inner_model.model_patcher.get_model_object("model_sampling")
     sigmas = offset_first_sigma_for_snr(sigmas, model_sampling)
     half_log_snrs = sigma_to_half_log_snr(sigmas, model_sampling)
-    er_lambdas = half_log_snrs.neg().exp()  # er_lambda_t = sigma_t / alpha_t
+    er_lambdas = half_log_snrs.neg().exp_()  # er_lambda_t = sigma_t / alpha_t
 
-    old_denoised = None
-    old_denoised_d = None
+    old_denoised = old_denoised_d = None
 
     for i in trange(len(sigmas) - 1, disable=disable):
-        denoised = model(x, sigmas[i] * s_in, **extra_args)
+        sigma, sigma_next = sigmas[i : i + 2]
+        denoised = model(x, sigma * s_in, **extra_args)
         if callback is not None:
-            callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
+            callback(
+                {
+                    "x": x,
+                    "i": i,
+                    "sigma": sigma,
+                    "sigma_hat": sigma,
+                    "denoised": denoised,
+                }
+            )
+        if sigma_next <= 0:
+            return denoised
+
         stage_used = min(max_stage, i + 1)
-        if sigmas[i + 1] == 0:
-            x = denoised
+        er_lambda_s, er_lambda_t = er_lambdas[i], er_lambdas[i + 1]
+
+        alpha_s = sigma / er_lambda_s
+        alpha_t = sigma_next / er_lambda_t
+        rho_sde_s = noise_scaler(er_lambda_s)
+        rho_sde_t = noise_scaler(er_lambda_t)
+        r_alpha = alpha_t / alpha_s
+        r_SDE = rho_sde_t / rho_sde_s
+        if solver_type == "sde":
+            r, r_sq = r_SDE, r_SDE**2
         else:
-            er_lambda_s, er_lambda_t = er_lambdas[i], er_lambdas[i + 1]
-            alpha_s = sigmas[i] / er_lambda_s
-            alpha_t = sigmas[i + 1] / er_lambda_t
-            r_alpha = alpha_t / alpha_s
-            r = noise_scaler(er_lambda_t) / noise_scaler(er_lambda_s)
+            r_ODE = er_lambda_t / er_lambda_s
+            r_sq = interpolation_function(r_ODE**2, r_SDE**2, eta**2).clamp_min_(0.0)
+            r = r_sq.sqrt()
 
-            # Stage 1 Euler
-            x = r_alpha * r * x + alpha_t * (1 - r) * denoised
+        # Stage 1 Euler
+        x = r_alpha * r * x + alpha_t * (1 - r) * denoised
 
-            if stage_used >= 2:
-                dt = er_lambda_t - er_lambda_s
-                lambda_step_size = -dt / num_integration_points
-                lambda_pos = er_lambda_t + point_indice * lambda_step_size
-                scaled_pos = noise_scaler(lambda_pos)
+        if stage_used >= 2:
+            dt = er_lambda_t - er_lambda_s
+            lambda_step_size = -dt / num_integration_points
+            lambda_pos = er_lambda_t + point_indice * lambda_step_size
+            scaled_pos = noise_scaler(lambda_pos)
 
-                # Stage 2
-                s = torch.sum(1 / scaled_pos) * lambda_step_size
-                denoised_d = (denoised - old_denoised) / (er_lambda_s - er_lambdas[i - 1])
-                x = x + alpha_t * (dt + s * noise_scaler(er_lambda_t)) * denoised_d
+            # Stage 2
+            s = (1 / scaled_pos).sum() * lambda_step_size
+            denoised_d = (denoised - old_denoised) / (er_lambda_s - er_lambdas[i - 1])
+            x += alpha_t * (dt + s * rho_sde_t) * denoised_d
 
-                if stage_used >= 3:
-                    # Stage 3
-                    s_u = torch.sum((lambda_pos - er_lambda_s) / scaled_pos) * lambda_step_size
-                    denoised_u = (denoised_d - old_denoised_d) / ((er_lambda_s - er_lambdas[i - 2]) / 2)
-                    x = x + alpha_t * ((dt ** 2) / 2 + s_u * noise_scaler(er_lambda_t)) * denoised_u
-                old_denoised_d = denoised_d
-
-            if s_noise > 0:
-                x = x + alpha_t * noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * (er_lambda_t ** 2 - er_lambda_s ** 2 * r ** 2).sqrt().nan_to_num(nan=0.0)
+            if stage_used >= 3:
+                # Stage 3
+                s_u = ((lambda_pos - er_lambda_s) / scaled_pos).sum() * lambda_step_size
+                denoised_u = (denoised_d - old_denoised_d) / (
+                    (er_lambda_s - er_lambdas[i - 2]) / 2
+                )
+                x += alpha_t * ((dt**2) / 2 + s_u * rho_sde_t) * denoised_u
+            old_denoised_d = denoised_d
         old_denoised = denoised
+
+        if eta <= 0:
+            continue
+
+        # When r approaches 0.0, noise_coeff approaches er_lambda_t (maximum possible added noise).
+        noise_coeff = (
+            (er_lambda_t**2 - er_lambda_s**2 * r_sq).sqrt_().nan_to_num_(nan=0.0)
+        )
+        x += alpha_t * noise_sampler(sigma, sigma_next) * s_noise * noise_coeff
     return x
 
 

comfy_api/latest/_input_impl/video_types.py

@@ -281,11 +281,18 @@ class VideoFromFile(VideoInput):
         video_done = False
         audio_done = True
 
-        if len(container.streams.audio):
-            audio_stream = container.streams.audio[-1]
+        # Use the last decodable audio stream. Streams FFmpeg has no decoder for have no codec context,
+        # and decoding their packets crashes the process. (e.g. APAC spatial-audio track in iPhone)
+        audio_stream = next(
+            (s for s in reversed(container.streams.audio) if s.codec_context is not None),
+            None,
+        )
+        if audio_stream is not None:
             streams += [audio_stream]
             resampler = av.audio.resampler.AudioResampler(format='fltp')
             audio_done = False
+        elif len(container.streams.audio):
+            logging.warning("No decodable audio stream found in video; ignoring audio.")
 
         for packet in container.demux(*streams):
             if video_done and audio_done:
@@ -457,10 +464,13 @@ class VideoFromFile(VideoInput):
                         else:
                             output_container.metadata[key] = json.dumps(value)
 
-                # Add streams to the new container
+                # Add streams to the new container. Streams with no codec context cannot be used as an output template.
                 stream_map = {}
                 for stream in streams:
                     if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)):
+                        if stream.codec_context is None:
+                            logging.warning("Skipping %s stream %d with unsupported codec", stream.type, stream.index)
+                            continue
                         out_stream = output_container.add_stream_from_template(template=stream, opaque=True)
                         stream_map[stream] = out_stream
 

comfy_api_nodes/util/upload_helpers.py

@@ -158,7 +158,14 @@ async def upload_video_to_comfyapi(
 
     # Convert VideoInput to BytesIO using specified container/codec
     video_bytes_io = BytesIO()
-    video.save_to(video_bytes_io, format=container, codec=codec)
+    try:
+        video.save_to(video_bytes_io, format=container, codec=codec)
+    except Exception as e:
+        raise ValueError(
+            f"Could not convert the input video to {container.value.upper()} for upload; "
+            f"the file may be corrupted or use an unsupported codec. "
+            f"Try re-exporting it as MP4 (H.264). Original error: {e}"
+        ) from e
     video_bytes_io.seek(0)
 
     return await upload_file_to_comfyapi(cls, video_bytes_io, filename, upload_mime_type, wait_label)

comfy_extras/nodes_custom_sampler.py

@@ -584,40 +584,102 @@ class SamplerDPMAdaptative(io.ComfyNode):
 
 class SamplerER_SDE(io.ComfyNode):
     @classmethod
-    def define_schema(cls):
+    def define_schema(cls) -> io.Schema:
         return io.Schema(
             node_id="SamplerER_SDE",
+            search_aliases=["sde", "er_sde", "ersde"],
             category="model/sampling/samplers",
             inputs=[
                 io.Combo.Input("solver_type", options=["ER-SDE", "Reverse-time SDE", "ODE"]),
-                io.Int.Input("max_stage", default=3, min=1, max=3, advanced=True),
-                io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, tooltip="Stochastic strength of reverse-time SDE.\nWhen eta=0, it reduces to deterministic ODE. This setting doesn't apply to ER-SDE solver type.", advanced=True),
-                io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True),
+                io.Int.Input(
+                    "max_stage",
+                    default=3,
+                    min=1,
+                    max=3,
+                    advanced=True,
+                    tooltip="Controls the number of stages the sampler uses. Stages: 1 - only uses the current step (Euler). 2 - Uses history from the previous step to improve accuracy. 3 - Uses two previous steps.",
+                ),
+                io.Float.Input(
+                    "eta",
+                    default=1.0,
+                    min=0.0,
+                    max=100.0,
+                    step=0.01,
+                    advanced=True,
+                    tooltip="Stochastic strength. Only has an effect when solver_type is not ODE.",
+                ),
+                io.Float.Input(
+                    "s_noise",
+                    default=1.0,
+                    min=-100.0,
+                    max=100.0,
+                    step=0.01,
+                    advanced=True,
+                    tooltip="SDE noise multiplier. Only has an effect when solver_type is not ODE.",
+                ),
+                io.Int.Input(
+                    "integration_points",
+                    default=200,
+                    min=1,
+                    max=10000,
+                    advanced=True,
+                    tooltip="More integration points improves accuracy with diminishing returns. The default is a good compromise. Only applies to the ER-SDE solver type.",
+                ),
+                io.Float.Input(
+                    "scaling_power",
+                    default=0.3,
+                    min=0.0,
+                    max=0.7,
+                    step=0.01,
+                    advanced=True,
+                    tooltip="Controls the exponent used for ER-SDE steps. Lower values make the sampler act more like a linear solver. Values above 0.5 may cause numerical overflow. Only has an effect when ETA is non-zero.",
+                ),
+                io.Float.Input(
+                    "scaling_constant",
+                    default=10.0,
+                    min=-0.99,
+                    max=100.0,
+                    step=0.1,
+                    advanced=True,
+                    tooltip="Constant value used for ER-SDE steps. Higher values cause the sampler to transition its stable, linear mode earlier while lower values will delay the transition.",
+                ),
             ],
-            outputs=[io.Sampler.Output()]
+            outputs=[io.Sampler.Output()],
         )
 
     @classmethod
-    def execute(cls, solver_type, max_stage, eta, s_noise) -> io.NodeOutput:
-        if solver_type == "ODE" or (solver_type == "Reverse-time SDE" and eta == 0):
-            eta = 0
-            s_noise = 0
-
-        def reverse_time_sde_noise_scaler(x):
-            return x ** (eta + 1)
-
-        if solver_type == "ER-SDE":
-            # Use the default one in sample_er_sde()
-            noise_scaler = None
+    def execute(
+        cls,
+        *,
+        solver_type: str,
+        max_stage: int,
+        eta: float,
+        s_noise: float,
+        integration_points: int,
+        scaling_power: float,
+        scaling_constant: float,
+    ) -> io.NodeOutput:
+        if solver_type == "ODE":
+            eta = s_noise = 0.0
+            solver_type = "sde"
+        elif solver_type == "Reverse-time SDE":
+            solver_type = "sde"
         else:
-            noise_scaler = reverse_time_sde_noise_scaler
-
-        sampler_name = "er_sde"
-        sampler = comfy.samplers.ksampler(sampler_name, {"s_noise": s_noise, "noise_scaler": noise_scaler, "max_stage": max_stage})
+            solver_type = "ersde"
+        sampler = comfy.samplers.ksampler(
+            "er_sde",
+            {
+                "solver_type": solver_type,
+                "eta": eta,
+                "s_noise": s_noise,
+                "max_stage": max_stage,
+                "num_integration_points": integration_points,
+                "scaling_power": scaling_power,
+                "scaling_constant": scaling_constant,
+            }
+        )
         return io.NodeOutput(sampler)
 
-    get_sampler = execute
-
 
 class SamplerSASolver(io.ComfyNode):
     @classmethod