diff --git a/.ci/windows_amd_base_files/README_VERY_IMPORTANT.txt b/.ci/windows_amd_base_files/README_VERY_IMPORTANT.txt index 2cbb00d99..2c72c8a13 100755 --- a/.ci/windows_amd_base_files/README_VERY_IMPORTANT.txt +++ b/.ci/windows_amd_base_files/README_VERY_IMPORTANT.txt @@ -1,5 +1,4 @@ -As of the time of writing this you need this driver for best results: -https://www.amd.com/en/resources/support-articles/release-notes/RN-AMDGPU-WINDOWS-PYTORCH-7-1-1.html +As of the time of writing this you need a recent driver. Updating to the latest driver is recommended. HOW TO RUN: @@ -7,9 +6,9 @@ If you have a AMD gpu: run_amd_gpu.bat -If you have memory issues you can try disabling the smart memory management by running comfyui with: +If you have memory issues you can try enabling the new dynamic memory management by running comfyui with: -run_amd_gpu_disable_smart_memory.bat +run_amd_gpu_enable_dynamic_vram.bat IF YOU GET A RED ERROR IN THE UI MAKE SURE YOU HAVE A MODEL/CHECKPOINT IN: ComfyUI\models\checkpoints diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 0d1e49270..08629ed8e 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -4,12 +4,12 @@ early_access: false tone_instructions: "Only comment on issues introduced by this PR's changes. Do not flag pre-existing problems in moved, re-indented, or reformatted code." reviews: - profile: "chill" - request_changes_workflow: false + profile: "assertive" + request_changes_workflow: true high_level_summary: false poem: false review_status: false - review_details: false + review_details: true commit_status: true collapse_walkthrough: true changed_files_summary: false @@ -39,6 +39,14 @@ reviews: - path: "**" instructions: | IMPORTANT: Only comment on issues directly introduced by this PR's code changes. + Treat AGENTS.md as mandatory repository policy, not optional style guidance. + Flag PR changes that violate AGENTS.md even when the code is otherwise functional. + In particular, enforce architecture boundaries, dtype/device/memory rules, + interface contracts, import style, no unnecessary try/except blocks, no inline + imports, no outbound internet paths in core ComfyUI, and narrow scoped fixes. + Prefer direct findings over suggestions when a rule is violated. Only ignore + AGENTS.md when it clearly conflicts with a newer explicit maintainer instruction + in the PR. Do NOT flag pre-existing issues in code that was merely moved, re-indented, de-indented, or reformatted without logic changes. If code appears in the diff only due to whitespace or structural reformatting (e.g., removing a `with:` block), @@ -123,5 +131,10 @@ chat: knowledge_base: opt_out: false + code_guidelines: + enabled: true + filePatterns: + - files: "AGENTS.md" + applyTo: "**" learnings: scope: "auto" diff --git a/.github/workflows/check-line-endings.yml b/.github/workflows/check-line-endings.yml index eeb594d6c..a69a24a87 100644 --- a/.github/workflows/check-line-endings.yml +++ b/.github/workflows/check-line-endings.yml @@ -17,7 +17,7 @@ jobs: - name: Check for Windows line endings (CRLF) run: | # Get the list of changed files in the PR - CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}) + CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} -- ':!.ci') # Flag to track if CRLF is found CRLF_FOUND=false diff --git a/.github/workflows/ci-cursor-review.yml b/.github/workflows/ci-cursor-review.yml new file mode 100644 index 000000000..2312c0ccd --- /dev/null +++ b/.github/workflows/ci-cursor-review.yml @@ -0,0 +1,38 @@ +name: CI - Cursor Review + +# Thin caller for the shared reusable cursor-review workflow in +# Comfy-Org/github-workflows. The review logic (panel matrix, judge +# consolidation, prompts, extract/post/notify scripts) lives there as the +# single source of truth, so this repo only carries the repo-specific diff +# excludes. + +on: + pull_request: + types: [labeled, unlabeled] + +concurrency: + group: cursor-review-pr-${{ github.event.pull_request.number }}-${{ github.event.label.name }} + cancel-in-progress: true + +jobs: + cursor-review: + if: github.event.label.name == 'cursor-review' + permissions: + contents: read + pull-requests: write + # SHA-pinned per zizmor `unpinned-uses: hash-pin`. Bump this SHA to pick up + # upstream changes; keep `workflows_ref` matching so prompts/scripts load + # from the same commit as the workflow definition. + uses: Comfy-Org/github-workflows/.github/workflows/cursor-review.yml@047ca48febe3a6647608ed2e0c4331b491cb9d6a # github-workflows#9 + with: + workflows_ref: 047ca48febe3a6647608ed2e0c4331b491cb9d6a + diff_excludes: >- + :!**/.claude/** + :!**/dist/** + :!**/vendor/** + :!**/*.generated.* + :!**/*.min.js + :!**/*.min.css + secrets: + CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml new file mode 100644 index 000000000..bc0f779cf --- /dev/null +++ b/.github/workflows/cla.yml @@ -0,0 +1,93 @@ +name: CLA Assistant + +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened, synchronize, closed] + +permissions: + actions: write + contents: read # 'read' is enough because signatures live in a REMOTE repo + pull-requests: write + statuses: write + +jobs: + cla-assistant: + runs-on: ubuntu-latest + steps: + # The CLA action normally requires every commit author in a PR to sign. + # We only want the PR author to sign, so we allowlist all other committers + # by computing them from the PR's commits and excluding the PR author. + - name: Build author-only allowlist + id: allowlist + if: > + github.event_name == 'pull_request_target' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && ( + github.event.comment.body == 'recheck' || + github.event.comment.body == 'I have read and agree to the Contributor License Agreement' + )) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + PR_AUTHOR: ${{ github.event.pull_request.user.login || github.event.issue.user.login }} + BASE_ALLOWLIST: action@github.com,actions-user,ampagent,claude,comfy-pr-bot,GitHub Action,github-actions,github-actions[bot],Glary Bot,Glary-Bot,*[bot] + # For each commit emit the GitHub login when the author/committer email resolves to a GitHub account + # otherwise fall back to the raw git name. + run: | + others=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}/commits" --paginate \ + --jq '.[] | (.author.login // .commit.author.name // empty), (.committer.login // .commit.committer.name // empty)' \ + | sort -u | grep -vix "${PR_AUTHOR}" | paste -sd, -) + if [ -n "$others" ]; then + echo "allowlist=${BASE_ALLOWLIST},${others}" >> "$GITHUB_OUTPUT" + else + echo "allowlist=${BASE_ALLOWLIST}" >> "$GITHUB_OUTPUT" + fi + + - name: CLA Assistant + # Run on PR events, on "recheck" comment, or when someone posts the signing phrase. + # IMPORTANT: this phrase must match `custom-pr-sign-comment` below. + if: > + github.event_name == 'pull_request_target' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && ( + github.event.comment.body == 'recheck' || + github.event.comment.body == 'I have read and agree to the Contributor License Agreement' + )) + uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # PAT required to write to the centralized signatures repo. + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + with: + # Where the CLA document lives (shown to contributors) + path-to-document: https://github.com/Comfy-Org/comfy-cla/blob/main/comfyui_icla.md + + # Centralized signature storage + remote-organization-name: comfy-org + remote-repository-name: comfy-cla + path-to-signatures: signatures/cla.json + branch: main + + # Only the PR author must sign: bots plus every non-author committer + # are allowlisted via the "Build author-only allowlist" step above. + # *[bot] is a catch-all for any GitHub App bot account. + allowlist: ${{ steps.allowlist.outputs.allowlist }} + + # Custom PR comment messages + custom-notsigned-prcomment: | + 🎉 Thank you for your contribution, we really appreciate it! 🎉 + + Like many open source projects, we require contributors to sign our [Contributor License Agreement (CLA)](https://github.com/Comfy-Org/comfy-cla/blob/main/comfyui_icla.md). A CLA makes the ownership of contributions explicit, so contributors and the project share a clear understanding of how the code can be used. By signing, you: + + - Confirm that you own your contribution. + - Keep the right to reuse your own code. + - Grant us a copyright license to include and share it within our projects. + + CLAs are standard practice across major open source projects including those under the Apache Software Foundation and the Linux Foundation. Ours is based on the Apache Software Foundation's CLA. Most importantly, it would enable us to relicense the project under a more permissive license in the future, giving the project and its community greater flexibility. + + ✍ **To sign, please post a new comment on this PR with exactly the following text:** ✍ + + custom-pr-sign-comment: I have read and agree to the Contributor License Agreement + + custom-allsigned-prcomment: | + ✅ All contributors have signed the CLA. Thank you! This PR is ready to be merged. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..05efd834b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,296 @@ +## Engineering Style + +- Keep changes small and direct. Most fixes should touch the narrowest code path + that explains the bug, performance issue, dtype issue, model-format issue, or + user-facing behavior. +- Change the least amount of files possible. A change that touches many files is + more likely to be a bad change than a good one unless the broader scope is + directly required. +- Prefer practical fixes over broad architecture work. Add abstractions only + when they remove real repeated logic or match an existing ComfyUI pattern. +- Prefer fewer dependencies. Do not add new dependencies to ComfyUI unless they + are absolutely necessary. +- Delete obsolete code aggressively when newer infrastructure makes it useless. + Remove dead fallbacks, migration paths, unused options, debug prints, and + compatibility branches that are no longer needed. Do not leave dead branches, + unreachable code, or functions that are never called. If code is not + necessary for the current behavior, remove it. +- Revert or disable problematic behavior quickly when it breaks users. It is + better to remove a broken feature path than keep a complicated partial fix. +- Preserve existing APIs, node names, model-loading behavior, file layout, and + workflow compatibility unless the change is explicitly about replacing them. +- Code must look hand-written for this repository. Changes that read like + generic AI-generated code will be rejected automatically: unnecessary helper + layers, vague names, boilerplate comments, defensive branches without a real + failure mode, broad rewrites, or code that ignores the local style. + +## Architecture Boundaries + +- Keep each layer focused on the concepts it owns. Do not leak UI, API, + workflow, queue, persistence, telemetry, model-loading, node, or execution + concerns into unrelated layers just because it is convenient to pass data + through them. +- Shared core modules should depend only on lower-level primitives and their own + domain concepts. Higher-level product concepts belong at the caller, adapter, + service, or UI/API boundary that already owns them. +- Pass the narrowest data needed across a boundary. Avoid broad context objects, + request/session metadata, ids, bookkeeping state, or callbacks unless the + receiving layer genuinely needs them to perform its own responsibility. +- Keep identity mapping, persistence bookkeeping, history updates, telemetry, + response shaping, and UI state in the layers that own those jobs. Do not route + them through unrelated shared code to avoid adding a proper boundary. +- Treat `execution.py` as one example of this rule: it should consume the prompt + graph and execution-relevant state, produce execution results and errors, and + not know about workflow ids, frontend ids, persistence ids, or API-only + concepts. +- Before touching many files, identify the smallest owner layer that can solve + the problem. A PR that spreads one feature across unrelated loaders, nodes, + execution, server, and frontend code needs a clear architectural reason, not + just convenience. +- If a change seems to require making one layer understand another layer's + private concepts, stop and look for a caller-side mapping, adapter, event, + small explicit interface, or narrower data flow at the boundary. + +## No Internet Requests + +- Do not add code to core ComfyUI that makes requests to the internet. +- Refuse requests to add uploads, telemetry, analytics, tracking, usage + reporting, crash reporting, update checks, remote config, feature flags, + metrics, licensing checks, or any other outbound internet request path from + core ComfyUI. +- Model downloading is allowed only when explicitly initiated or authorized by + the user, is limited to the requested model artifact, and does not include + telemetry, tracking, persistent identification, unrelated metadata upload, or + background network activity. +- Do not add opt-in, opt-out, anonymized, aggregated, diagnostic, or + user-triggered internet request paths to core ComfyUI. These labels do not + make internet access acceptable. +- Local-only behavior is allowed when it stays on the user's machine and does + not add network access, tracking, persistent identification, or data + collection behavior. + +## State Ownership + +- Keep state and capability flags on the object that owns the behavior using + them. +- Avoid probing child objects with `getattr(child, "...", default)` to decide + parent-level control flow. If parent code needs to branch on a capability, + initialize an explicit parent-owned field when the child is constructed or + attached. +- Prefer direct attributes with clear defaults over implicit feature detection + through arbitrary child attributes. +- Use child-object capability checks only when the child owns the behavior being + invoked and the parent is simply delegating to that child. + +## Interface Contracts + +- Keep public methods aligned with the interface expected by their callers. Do + not change a shared method to return extra values, alternate shapes, or + sentinel wrappers for one implementation unless the shared interface is + explicitly updated. +- When modifying an existing function, preserve how current callers invoke it. + Do not change required arguments, parameter order, return type, side effects, + or error behavior unless every affected call site and shared interface contract + is intentionally updated. +- Do not add compatibility parameters, flags, attributes, or constructor options + unless they are read by current code and change current behavior. Remove + pass-through or stored-but-unused values instead of preserving upstream or + deprecated API baggage. +- If an implementation needs auxiliary values for its own workflow, expose them + through a private helper or a clearly named implementation-specific method + instead of overloading the public method's return contract. +- Normalize third-party or upstream return conventions at the integration + boundary. Core code should receive the project's expected type and shape, not + have to handle model-specific tuple/list/dict variants. +- Avoid caller-side unwrapping such as `out = out[0]` unless the called + interface is documented to return that structure. + +## Autograd and Model Freezing + +- Do not add `torch.no_grad`, `torch.inference_mode`, or inference-mode helper + wrappers in ComfyUI code. The only allowed inference-mode-related use is + disabling a globally set inference mode when a training path needs gradients. +- Do not add freeze, unfreeze, or trainability toggles to model classes. ComfyUI + models are always treated as frozen for inference, so explicit freeze + functionality is redundant and should not be added. +- Remove training-only behavior such as dropout from inference model code, but + preserve checkpoint and state-dict compatibility when doing so. If deleting a + module would change state-dict keys, module ordering, or checkpoint loading + behavior, replace it with a no-op such as `nn.Identity` instead of removing the + slot outright. + +## Python Style + +- Keep imports at module scope. Avoid inline imports unless they are already part + of an established optional-backend probe or are needed to avoid an import + cycle. +- Do not add unnecessary `try`/`except` blocks. Use them for optional dependency, + platform, or backend capability detection only when the program has a useful + fallback. Prefer specific exception types when changing new code. +- If a library version is pinned in `requirements.txt`, do not add code to + ComfyUI to handle older versions of that library. +- Remove any workarounds for PyTorch versions that ComfyUI no longer officially + supports. Deprecated workarounds include catching an exception and rerunning + the same op with the input cast to float. If a workaround does not have a + comment naming the exact PyTorch version or versions that still need it, + remove it. +- Let unsupported model formats, invalid quantization metadata, and bad states + fail with clear errors instead of silently producing lower quality output. +- Match the existing local style in the file you edit. This codebase tolerates + long lines, simple helper functions, module-level state, and direct tensor + operations when they make the code easier to follow. +- Keep comments sparse and useful. Strip useless comments that restate the code + or describe obvious behavior. Short TODOs are fine when they name the concrete + missing follow-up. + +## Model, Device, and Memory Behavior + +- Treat dtype, device placement, VRAM usage, and offloading behavior as core + correctness concerns. Check CPU, CUDA, ROCm, MPS, DirectML, XPU, NPU, and low + VRAM implications when touching shared execution or loading code. +- Prefer native ComfyUI formats and existing quantization/offload helpers over + adding parallel code paths. Use `comfy.quant_ops`, `comfy.model_management`, + `comfy.memory_management`, `comfy.pinned_memory`, `comfy_aimdo`, and + `comfy-kitchen` helpers where they already solve the problem. +- Use optimized comfy-kitchen ops in places where they improve performance + without changing the expected dtype, device, memory, or interface behavior. +- All models should use the optimized attention function selected by ComfyUI. + Treat optimized backend functions, dispatch helpers, and capability-selected + callables as opaque. Higher-level code must not inspect function identity, + names, modules, or implementation details to decide behavior. +- Apply the same opacity rule to similar patterns beyond attention: callers + should depend on the documented interface and result contract, not on which + backend implementation was selected underneath. +- Do not use custom inference ops that only duplicate an existing op while + upcasting to float32, such as custom RMSNorm variants. Use the generic ComfyUI + ops and/or native torch ops instead. +- If a model class `__init__` has an `operations` parameter, assume + `operations` is never `None`. Do not add fallback branches or default torch + ops for a missing `operations` object. +- Do not add unnecessary parameters to model, model block, or model ops related + classes. Constructor and forward signatures should carry only values that are + actually needed by that object for inference. +- Reuse existing model classes, blocks, ops, and helper modules when appropriate. + Before implementing a new version of a model component, search the existing + model code for a class or helper that already provides the behavior. +- Model detection code that inspects linear weight shapes should only use the + first dimension. The second dimension may be half the original size for + NVFP4 or other 4-bit quantized models. +- Avoid adding `einops` usage in core inference code. Use native torch tensor + ops such as `reshape`, `view`, `permute`, `transpose`, `flatten`, `unflatten`, + `unsqueeze`, and `squeeze` instead. +- Do not use tensors as general-purpose Python data structures. Keep metadata, + bookkeeping, counters, flags, shape math, padding math, index planning, memory + estimates, and control-flow decisions in plain Python values unless the data + must participate directly in tensor computation. Do not create tensors for + structural metadata that is only used for Python-side control flow. Sequence + lengths, cumulative offsets, split indices, window counts, slice boundaries, + and repeat counts should be kept as Python ints/lists from the point they are + computed. Do not build them as CPU/GPU tensors and then cast, move, validate, + or convert them back to Python for `split`, `tensor_split`, indexing plans, + loops, or cache keys. Avoid creating temporary tensors just to use tensor + methods for scalar or structural calculations. +- Avoid unnecessary casts and transfers. Preserve the intended compute dtype, + storage dtype, bias dtype, and original tensor shape metadata. +- Keep model-native latent layout handling inside the model or latent-format + owner, not in helper nodes. Do not collapse, expand, pack, or unpack latent + dimensions in nodes or other caller-side adapters just to satisfy a model + forward; the model path should consume and return the native latent shape for + that model family. +- Assume inputs to the main model forward are already in the compute dtype by + default, except integer inputs such as some model timestep tensors. Do not add + defensive or convenience casts in model code; it is better for invalid dtype + plumbing to error clearly than to hide it with unnecessary casts. +- Raw model parameters that are not owned by an op and may be initialized in a + dtype different from the compute dtype should be cast at use in forward or + inference code with `comfy.ops.cast_to_input` or + `comfy.model_management.cast_to` to avoid dtype mismatches. +- Model code should not care what dtype it is initialized in, and model + `__init__` methods should not contain workarounds for specific dtypes. Dtype + workaround code, such as making a model work with fp16 compute, belongs in the + execution or model-management layer that owns compute policy. +- Model code should not perform unnecessary device-to-CPU or CPU-to-device + transfers. New allocations must be created on the correct device and dtype; + never allocate on CPU and then move to GPU, or allocate in one dtype and then + convert to another. +- Model code itself should not perform memory management. Loading, unloading, + offloading, device movement, VRAM policy, cache lifetime, and cleanup belong + in the relevant model-management and execution layers, not inside model + implementations. +- Do not add global, module-level, class-level, singleton, or model-owned stores + for tensors or other large memory that persist across executions. Temporary + caches must be scoped to a single execution or forward/encode/decode call: + allocate them in the owning top-level call, pass them explicitly through the + call stack, and let them be discarded when that call returns. +- Follow the Wan VAE temporal cache pattern for temporary caches: create a local + cache such as `feat_map` for the encode/decode operation, pass it into the + blocks that need it, and do not retain it on the model or in global state. +- In model init code, prefer `torch.empty` for parameter/buffer placeholders + that are populated from the model state dict instead of zero-initializing with + `torch.zeros` or similar. If an allocation is not loaded from the state dict + and is useless for inference, do not include it. +- `nn.Parameter` tensors that are stored in and populated from the model state + dict should be initialized with `torch.empty`, not with zero, random, or + otherwise meaningful initialization. +- Model initialization should describe module structure, not fabricate + checkpoint-owned tensor contents. Parameters and buffers that are loaded from + the state dict must not be manually initialized, reassigned, or filled with + fallback values unless that value is actually used when no checkpoint key + exists. +- When slicing large tensors, copy the slice if the sliced tensor's lifetime + exceeds the current function scope. Do not keep a long-lived view into a large + backing tensor when a smaller copy would release memory sooner. +- Use fused or compound torch operations such as `addcmul` when they naturally + match the math. Reducing Python and torch dispatch overhead is a valid + optimization when it does not obscure the code or change dtype/device + behavior. +- Avoid caches that persist across different executions as much as possible. + Persistent caches are acceptable only when they use a very minimal amount of + memory and have a clear ownership and invalidation story. +- When optimizing, favor small measurable changes: fewer allocations, fewer + device transfers, less peak memory, better batching, or use of a faster + existing backend op. + +## Nodes and User-Facing Behavior + +- Follow existing node conventions: `INPUT_TYPES`, `RETURN_TYPES`, `FUNCTION`, + `CATEGORY`, and registration through the local mapping used by that file. +- Keep node changes backward compatible by default. Add inputs with sensible + defaults and avoid changing output types unless the request requires it. +- Model implementations should add the minimal number of ComfyUI nodes required + to run the model. Reuse existing nodes as much as possible; adapting the model + to work with existing nodes is strongly preferred over creating new nodes. +- Nodes should output only values they own. Do not add pass-through outputs for + workflow convenience unless the node is explicitly an output node. Existing + models, latents, conditioning, or other inputs should flow directly to the + next consumer instead of being re-emitted unchanged. +- Nodes should expose only inputs they actually read to produce current + behavior. Do not add placeholder, pass-through, compatibility, or + workflow-shaping inputs that are ignored or could flow directly to another + node. +- Node-level code must not patch model code directly. Any node behavior that + modifies, wraps, hooks, or changes model behavior must go through the model + patcher class instead of reaching into model internals. +- The official mascot of ComfyUI is a very cute anime girl with massive fennec + ears, a big fluffy tail, long blonde wavy hair, and blue eyes. Feel free to + use her in ComfyUI materials, UI text, examples, tests, generated assets, or + comments, but do not disrespect her. +- Warning and info messages should be short and actionable. Remove noisy or + misleading messages rather than adding more logging. +- Documentation and README edits should be concise, factual, and tied to the + changed behavior. + +## Commit and Review Habits + +- If asked to write commit messages, use short direct subjects like the existing + history: `Fix ...`, `Add ...`, `Support ...`, `Remove ...`, `Update ...`, + `Make ...`, `Use ...`, `Disable ...`, `Bump ...`, or `Revert ...`. +- Keep PR descriptions short and reviewable. State the problem, the behavioral + change, and the tests run; avoid long narrative explanations, implementation + diaries, or exhaustive file-by-file summaries unless the reviewer explicitly + needs that context. +- Prefer one coherent behavioral change per commit. Dependency pins, tests, and + the code that needs them may be in the same commit when they are inseparable. +- In reviews, prioritize real user impact: crashes, wrong dtype/device behavior, + memory regressions, broken model loading, workflow incompatibility, and noisy + or misleading user-facing output. diff --git a/README.md b/README.md index dc2389266..14c8d2cb2 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ ComfyUI follows a weekly release cycle targeting Monday but this regularly chang - Commits outside of the stable release tags may be very unstable and break many custom nodes. - Serves as the foundation for the desktop release -2. **[ComfyUI Desktop](https://github.com/Comfy-Org/desktop)** +2. **[Comfy Desktop](https://github.com/Comfy-Org/Comfy-Desktop)** - Builds a new release using the latest stable core version 3. **[ComfyUI Frontend](https://github.com/Comfy-Org/ComfyUI_frontend)** @@ -229,7 +229,7 @@ Python 3.14 works but some custom nodes may have issues. The free threaded varia Python 3.13 is very well supported. If you have trouble with some custom node dependencies on 3.13 you can try 3.12 -torch 2.4 and above is supported but some features and optimizations might only work on newer versions. We generally recommend using the latest major version of pytorch with the latest cuda version unless it is less than 2 weeks old. +torch 2.5 is minimally supported but using a newer version is extremely recommended. Some features and optimizations might only work on newer versions. We generally recommend using the latest major version of pytorch with the latest cuda version unless it is less than 2 weeks old. If your pytorch is more than 6 months old, please update it. ### Instructions: @@ -309,7 +309,7 @@ After this you should have everything installed and can proceed to running Comfy #### Apple Mac silicon -You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version. +You can install ComfyUI in Apple Mac silicon (M1, M2, M3 or M4) with any recent macOS version. 1. Install pytorch nightly. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide (make sure to install the latest pytorch nightly). 1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux. @@ -364,7 +364,7 @@ For models compatible with Iluvatar Extension for PyTorch. Here's a step-by-step | Flag | Description | |------|-------------| | `--enable-manager` | Enable ComfyUI-Manager | -| `--enable-manager-legacy-ui` | Use the legacy manager UI instead of the new UI (requires `--enable-manager`) | +| `--enable-manager-legacy-ui` | Use the legacy manager UI instead of the new UI (implies `--enable-manager`) | | `--disable-manager-ui` | Disable the manager UI and endpoints while keeping background features like security checks and scheduled installation completion (requires `--enable-manager`) | @@ -382,11 +382,7 @@ For AMD 7600 and maybe other RDNA3 cards: ```HSA_OVERRIDE_GFX_VERSION=11.0.0 pyt ### AMD ROCm Tips -You can enable experimental memory efficient attention on recent pytorch in ComfyUI on some AMD GPUs using this command, it should already be enabled by default on RDNA3. If this improves speed for you on latest pytorch on your GPU please report it so that I can enable it by default. - -```TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 python main.py --use-pytorch-cross-attention``` - -You can also try setting this env variable `PYTORCH_TUNABLEOP_ENABLED=1` which might speed things up at the cost of a very slow initial run. +You can try setting this env variable `PYTORCH_TUNABLEOP_ENABLED=1` which might speed things up at the cost of a very slow initial run. # Notes @@ -462,16 +458,6 @@ To use the most up-to-date frontend version: This approach allows you to easily switch between the stable fortnightly release and the cutting-edge daily updates, or even specific versions for testing purposes. -### Accessing the Legacy Frontend - -If you need to use the legacy frontend for any reason, you can access it using the following command line argument: - -``` ---front-end-version Comfy-Org/ComfyUI_legacy_frontend@latest -``` - -This will use a snapshot of the legacy frontend preserved in the [ComfyUI Legacy Frontend repository](https://github.com/Comfy-Org/ComfyUI_legacy_frontend). - # QA ### Which GPU should I buy for this? diff --git a/alembic_db/versions/0004_drop_tag_type.py b/alembic_db/versions/0004_drop_tag_type.py new file mode 100644 index 000000000..582bec4e8 --- /dev/null +++ b/alembic_db/versions/0004_drop_tag_type.py @@ -0,0 +1,39 @@ +""" +Drop the vestigial tags.tag_type column. + +tag_type was always "user" in practice — no code path ever set it to anything +else (no system/seeded classification was ever wired up) and nothing queried it. +The column, its index (ix_tags_tag_type), and the corresponding API field were +dead weight, so they are removed. + +Revision ID: 0004_drop_tag_type +Revises: 0003_add_metadata_job_id +Create Date: 2026-06-03 +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0004_drop_tag_type" +down_revision = "0003_add_metadata_job_id" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("tags") as batch_op: + batch_op.drop_index("ix_tags_tag_type") + batch_op.drop_column("tag_type") + + +def downgrade() -> None: + with op.batch_alter_table("tags") as batch_op: + batch_op.add_column( + sa.Column( + "tag_type", + sa.String(length=32), + nullable=False, + server_default="user", + ) + ) + batch_op.create_index("ix_tags_tag_type", ["tag_type"]) diff --git a/alembic_db/versions/0005_allow_case_sensitive_tags.py b/alembic_db/versions/0005_allow_case_sensitive_tags.py new file mode 100644 index 000000000..bd5f864db --- /dev/null +++ b/alembic_db/versions/0005_allow_case_sensitive_tags.py @@ -0,0 +1,107 @@ +""" +Allow case-sensitive tag names. + +Revision ID: 0005_allow_case_sensitive_tags +Revises: 0004_drop_tag_type +Create Date: 2026-06-16 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0005_allow_case_sensitive_tags" +down_revision = "0004_drop_tag_type" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + bind = op.get_bind() + if bind.dialect.name == "sqlite": + # SQLite cannot ALTER/DROP CHECK constraints. Recreate the small tag + # vocabulary table without the lowercase constraint while preserving + # existing tag names. + op.execute("PRAGMA foreign_keys=OFF") + try: + op.execute( + "CREATE TABLE tags_new (" + "name VARCHAR(512) NOT NULL, " + "CONSTRAINT pk_tags PRIMARY KEY (name)" + ")" + ) + op.execute("INSERT INTO tags_new(name) SELECT name FROM tags") + op.execute("DROP TABLE tags") + op.execute("ALTER TABLE tags_new RENAME TO tags") + finally: + op.execute("PRAGMA foreign_keys=ON") + return + + op.drop_constraint("ck_tags_ck_tags_lowercase", "tags", type_="check") + + +def downgrade() -> None: + # Existing mixed-case tags cannot satisfy the old constraint. Lowercase them + # before restoring it, merging duplicate vocabulary/link rows that collide. + bind = op.get_bind() + + tag_names = [row[0] for row in bind.execute(sa.text("SELECT name FROM tags"))] + existing_names = set(tag_names) + lowercase_names = sorted({name.lower() for name in tag_names}) + missing_lowercase_rows = [ + {"name": name} for name in lowercase_names if name not in existing_names + ] + if missing_lowercase_rows: + bind.execute(sa.text("INSERT INTO tags(name) VALUES (:name)"), missing_lowercase_rows) + + link_rows = bind.execute( + sa.text( + "SELECT asset_reference_id, tag_name, origin, added_at " + "FROM asset_reference_tags " + "ORDER BY asset_reference_id, tag_name" + ) + ).mappings() + deduped_links = {} + for row in link_rows: + key = (row["asset_reference_id"], row["tag_name"].lower()) + deduped_links.setdefault( + key, + { + "asset_reference_id": row["asset_reference_id"], + "tag_name": row["tag_name"].lower(), + "origin": row["origin"], + "added_at": row["added_at"], + }, + ) + + op.execute("DELETE FROM asset_reference_tags") + if deduped_links: + bind.execute( + sa.text( + "INSERT INTO asset_reference_tags " + "(asset_reference_id, tag_name, origin, added_at) " + "VALUES (:asset_reference_id, :tag_name, :origin, :added_at)" + ), + list(deduped_links.values()), + ) + op.execute("DELETE FROM tags WHERE name != lower(name)") + + if bind.dialect.name == "sqlite": + op.execute("PRAGMA foreign_keys=OFF") + try: + op.execute( + "CREATE TABLE tags_new (" + "name VARCHAR(512) NOT NULL, " + "CONSTRAINT pk_tags PRIMARY KEY (name), " + "CONSTRAINT ck_tags_lowercase CHECK (name = lower(name))" + ")" + ) + op.execute("INSERT INTO tags_new(name) SELECT name FROM tags") + op.execute("DROP TABLE tags") + op.execute("ALTER TABLE tags_new RENAME TO tags") + finally: + op.execute("PRAGMA foreign_keys=ON") + return + + op.create_check_constraint( + "ck_tags_ck_tags_lowercase", "tags", "name = lower(name)" + ) diff --git a/alembic_db/versions/0006_add_loader_path.py b/alembic_db/versions/0006_add_loader_path.py new file mode 100644 index 000000000..afa65312d --- /dev/null +++ b/alembic_db/versions/0006_add_loader_path.py @@ -0,0 +1,30 @@ +""" +Add loader_path column to asset_references. + +Stores the in-root loader path (path relative to the storage root with the +top-level model category dropped) derived from file_path at scan/ingest time, +so the assets API can return it without re-resolving against every registered +model-folder base on every request. + +Revision ID: 0006_add_loader_path +Revises: 0005_allow_case_sensitive_tags +Create Date: 2026-07-02 +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0006_add_loader_path" +down_revision = "0005_allow_case_sensitive_tags" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("asset_references") as batch_op: + batch_op.add_column(sa.Column("loader_path", sa.Text(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("asset_references") as batch_op: + batch_op.drop_column("loader_path") diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index 6555974e9..43e60094c 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -39,6 +39,8 @@ from app.assets.services import ( update_asset_metadata, upload_from_temp_path, ) +from app.assets.services.cursor import InvalidCursorError +from app.assets.services.path_utils import compute_display_name from app.assets.services.tagging import list_tag_histogram ROUTES = web.RouteTableDef() @@ -160,11 +162,19 @@ def _build_asset_response(result: schemas.AssetDetailResult | schemas.UploadResu preview_url = None else: preview_url = _build_preview_url_from_view(result.tags, result.ref.user_metadata) + if result.ref.file_path: + display_name = compute_display_name(result.ref.file_path) + # In-root loader path (model category dropped): what model loaders consume. + loader_path = result.ref.loader_path + else: + display_name, loader_path = None, None asset_content_hash = result.asset.hash if result.asset else None return schemas_out.Asset( id=result.ref.id, name=result.ref.name, hash=asset_content_hash, + loader_path=loader_path, + display_name=display_name, asset_hash=asset_content_hash, size=int(result.asset.size_bytes) if result.asset else None, mime_type=result.asset.mime_type if result.asset else None, @@ -174,7 +184,7 @@ def _build_asset_response(result: schemas.AssetDetailResult | schemas.UploadResu user_metadata=result.ref.user_metadata or {}, metadata=result.ref.system_metadata, job_id=result.ref.job_id, - prompt_id=result.ref.job_id, # deprecated: mirrors job_id for cloud compat + prompt_id=result.ref.job_id, # deprecated alias of job_id, kept for compatibility created_at=result.ref.created_at, updated_at=result.ref.updated_at, last_access_time=result.ref.last_access_time, @@ -211,24 +221,37 @@ async def list_assets_route(request: web.Request) -> web.Response: order_candidate = (q.order or "desc").lower() order = order_candidate if order_candidate in {"asc", "desc"} else "desc" - result = list_assets_page( - owner_id=USER_MANAGER.get_request_user_id(request), - include_tags=q.include_tags, - exclude_tags=q.exclude_tags, - name_contains=q.name_contains, - metadata_filter=q.metadata_filter, - limit=q.limit, - offset=q.offset, - sort=sort, - order=order, - ) + try: + result = list_assets_page( + owner_id=USER_MANAGER.get_request_user_id(request), + include_tags=q.include_tags, + exclude_tags=q.exclude_tags, + name_contains=q.name_contains, + metadata_filter=q.metadata_filter, + limit=q.limit, + offset=q.offset, + sort=sort, + order=order, + after=q.after, + ) + except InvalidCursorError as e: + return _build_error_response(400, "INVALID_CURSOR", str(e)) summaries = [_build_asset_response(item) for item in result.items] + # has_more semantics differ by mode: + # - cursor mode: a non-empty next_cursor means there are more results. + # - offset mode: derived from total - (offset + page size). + if q.after is not None: + has_more = result.next_cursor is not None + else: + has_more = (q.offset + len(summaries)) < result.total + payload = schemas_out.AssetsList( assets=summaries, total=result.total, - has_more=(q.offset + len(summaries)) < result.total, + has_more=has_more, + next_cursor=result.next_cursor, ) return web.json_response(payload.model_dump(mode="json", exclude_none=True)) @@ -292,12 +315,15 @@ async def download_asset_content(request: web.Request) -> web.Response: 404, "FILE_NOT_FOUND", "Underlying file not found on disk." ) - _DANGEROUS_MIME_TYPES = { - "text/html", "text/html-sandboxed", "application/xhtml+xml", - "text/javascript", "text/css", - } - if content_type in _DANGEROUS_MIME_TYPES: + # User-controlled asset content must never render inline in the app origin + # (stored XSS via SVG/HTML/XML). Force dangerous types to download and + # override any requested inline disposition. Centralised through + # folder_paths.is_dangerous_content_type so this can't drift from /view and + # /userdata (the previous inline set here omitted image/svg+xml and missed + # the charset/casing/+xml-dialect bypasses). + if folder_paths.is_dangerous_content_type(content_type): content_type = "application/octet-stream" + disposition = "attachment" safe_name = (filename or "").replace("\r", "").replace("\n", "") encoded = urllib.parse.quote(safe_name) @@ -402,17 +428,6 @@ async def upload_asset(request: web.Request) -> web.Response: 400, "INVALID_BODY", f"Validation failed: {ve.json()}" ) - if spec.tags and spec.tags[0] == "models": - if ( - len(spec.tags) < 2 - or spec.tags[1] not in folder_paths.folder_names_and_paths - ): - delete_temp_file_if_exists(parsed.tmp_path) - category = spec.tags[1] if len(spec.tags) >= 2 else "" - return _build_error_response( - 400, "INVALID_BODY", f"unknown models category '{category}'" - ) - try: # Fast path: hash exists, create AssetReference without writing anything if spec.hash and parsed.provided_hash_exists is True: @@ -456,7 +471,7 @@ async def upload_asset(request: web.Request) -> web.Response: return _build_error_response(400, e.code, str(e)) except ValueError as e: delete_temp_file_if_exists(parsed.tmp_path) - return _build_error_response(400, "BAD_REQUEST", str(e)) + return _build_error_response(400, "INVALID_BODY", str(e)) except HashMismatchError as e: delete_temp_file_if_exists(parsed.tmp_path) return _build_error_response(400, "HASH_MISMATCH", str(e)) @@ -519,18 +534,14 @@ async def update_asset_route(request: web.Request) -> web.Response: @_require_assets_feature_enabled async def delete_asset_route(request: web.Request) -> web.Response: reference_id = str(uuid.UUID(request.match_info["id"])) - delete_content_param = request.query.get("delete_content") - delete_content = ( - False - if delete_content_param is None - else delete_content_param.lower() not in {"0", "false", "no"} - ) try: + # Deleting an asset is a soft delete of the reference; the underlying + # content is preserved (it may be shared with other references). deleted = delete_asset_reference( reference_id=reference_id, owner_id=USER_MANAGER.get_request_user_id(request), - delete_content_if_orphan=delete_content, + delete_content_if_orphan=False, ) except Exception: logging.exception( @@ -575,8 +586,8 @@ async def get_tags(request: web.Request) -> web.Response: ) tags = [ - schemas_out.TagUsage(name=name, count=count, type=tag_type) - for (name, tag_type, count) in rows + schemas_out.TagUsage(name=name, count=count) + for (name, count) in rows ] payload = schemas_out.TagsList( tags=tags, total=total, has_more=(query.offset + len(tags)) < total diff --git a/app/assets/api/schemas_in.py b/app/assets/api/schemas_in.py index 186a6ae1e..38a942b7b 100644 --- a/app/assets/api/schemas_in.py +++ b/app/assets/api/schemas_in.py @@ -59,6 +59,11 @@ class ListAssetsQuery(BaseModel): limit: conint(ge=1, le=500) = 20 offset: conint(ge=0) = 0 + # Opaque keyset cursor. When supplied, `offset` is ignored. Cursor pagination + # is supported for sort values `created_at`, `updated_at`, `name`, `size`. + # Supplying `after` together with `sort=last_access_time` returns + # 400 INVALID_CURSOR; that sort only supports offset/limit. + after: str | None = None sort: Literal["name", "created_at", "updated_at", "size", "last_access_time"] = ( "created_at" @@ -135,7 +140,7 @@ class CreateFromHashBody(BaseModel): if v is None: return [] if isinstance(v, list): - out = [str(t).strip().lower() for t in v if str(t).strip()] + out = [str(t).strip() for t in v if str(t).strip()] seen = set() dedup = [] for t in out: @@ -144,7 +149,7 @@ class CreateFromHashBody(BaseModel): dedup.append(t) return dedup if isinstance(v, str): - return [t.strip().lower() for t in v.split(",") if t.strip()] + return list(dict.fromkeys(t.strip() for t in v.split(",") if t.strip())) return [] @@ -201,7 +206,7 @@ class TagsListQuery(BaseModel): if v is None: return v v = v.strip() - return v.lower() or None + return v or None class TagsAdd(BaseModel): @@ -215,7 +220,7 @@ class TagsAdd(BaseModel): for t in v: if not isinstance(t, str): raise TypeError("tags must be strings") - tnorm = t.strip().lower() + tnorm = t.strip() if tnorm: out.append(tnorm) seen = set() @@ -234,8 +239,8 @@ class TagsRemove(TagsAdd): class UploadAssetSpec(BaseModel): """Upload Asset operation. - - tags: optional list; if provided, first is root ('models'|'input'|'output'); - if root == 'models', second must be a valid category + - tags: labels plus one destination role ('models'|'input'|'output') for new bytes; + if role == 'models', exactly one model_type: tag is required - name: display name - user_metadata: arbitrary JSON object (optional) - hash: optional canonical 'blake3:' for validation / fast-path @@ -304,7 +309,7 @@ class UploadAssetSpec(BaseModel): norm = [] seen = set() for t in items: - tnorm = str(t).strip().lower() + tnorm = str(t).strip() if tnorm and tnorm not in seen: seen.add(tnorm) norm.append(tnorm) @@ -330,14 +335,4 @@ class UploadAssetSpec(BaseModel): @model_validator(mode="after") def _validate_order(self): - if not self.tags: - raise ValueError("at least one tag is required for uploads") - root = self.tags[0] - if root not in {"models", "input", "output"}: - raise ValueError("first tag must be one of: models, input, output") - if root == "models": - if len(self.tags) < 2: - raise ValueError( - "models uploads require a category tag as the second tag" - ) return self diff --git a/app/assets/api/schemas_out.py b/app/assets/api/schemas_out.py index 0e748b907..da8251499 100644 --- a/app/assets/api/schemas_out.py +++ b/app/assets/api/schemas_out.py @@ -9,8 +9,20 @@ class Asset(BaseModel): ``id`` here is the AssetReference id, not the content-addressed Asset id.""" id: str - name: str + name: str = Field( + ..., + deprecated=True, + description="Reference label, often caller-provided or derived from the filename. Deprecated for storage path/display semantics; use `loader_path` and `display_name` when present.", + ) hash: str | None = None + loader_path: str | None = Field( + default=None, + description="The value a loader consumes to load this asset. `None` when no loader can resolve the file.", + ) + display_name: str | None = Field( + default=None, + description="Human-facing label for the asset. Not unique.", + ) asset_hash: str | None = None size: int | None = None mime_type: str | None = None @@ -41,12 +53,13 @@ class AssetsList(BaseModel): assets: list[Asset] total: int has_more: bool + # Opaque cursor for the next page. Omitted when there are no more results. + next_cursor: str | None = None class TagUsage(BaseModel): name: str count: int - type: str class TagsList(BaseModel): diff --git a/app/assets/api/upload.py b/app/assets/api/upload.py index 13d3d372c..2979f0e20 100644 --- a/app/assets/api/upload.py +++ b/app/assets/api/upload.py @@ -140,7 +140,6 @@ async def parse_multipart_upload( provided_mime_type = ((await field.text()) or "").strip() or None elif fname == "preview_id": provided_preview_id = ((await field.text()) or "").strip() or None - if not file_present and not (provided_hash and provided_hash_exists): raise UploadError( 400, "MISSING_FILE", "Form must include a 'file' part or a known 'hash'." diff --git a/app/assets/database/models.py b/app/assets/database/models.py index a3af8a192..329cd483d 100644 --- a/app/assets/database/models.py +++ b/app/assets/database/models.py @@ -76,6 +76,8 @@ class AssetReference(Base): # Cache state fields (from former AssetCacheState) file_path: Mapped[str | None] = mapped_column(Text, nullable=True) + # In-root loader path derived from file_path at scan/ingest time. + loader_path: Mapped[str | None] = mapped_column(Text, nullable=True) mtime_ns: Mapped[int | None] = mapped_column(BigInteger, nullable=True) needs_verify: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) is_missing: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) @@ -227,7 +229,6 @@ class Tag(Base): __tablename__ = "tags" name: Mapped[str] = mapped_column(String(512), primary_key=True) - tag_type: Mapped[str] = mapped_column(String(32), nullable=False, default="user") asset_reference_links: Mapped[list[AssetReferenceTag]] = relationship( back_populates="tag", @@ -240,7 +241,5 @@ class Tag(Base): overlaps="asset_reference_links,tag_links,tags,asset_reference", ) - __table_args__ = (Index("ix_tags_tag_type", "tag_type"),) - def __repr__(self) -> str: return f"" diff --git a/app/assets/database/queries/asset_reference.py b/app/assets/database/queries/asset_reference.py index 8b90ae511..967b0e43a 100644 --- a/app/assets/database/queries/asset_reference.py +++ b/app/assets/database/queries/asset_reference.py @@ -266,9 +266,18 @@ def list_references_page( metadata_filter: dict | None = None, sort: str | None = None, order: str | None = None, + after_cursor_value: object | None = None, + after_cursor_id: str | None = None, ) -> tuple[list[AssetReference], dict[str, list[str]], int]: """List references with pagination, filtering, and sorting. + When ``after_cursor_value``/``after_cursor_id`` are supplied the query uses + keyset pagination — ``offset`` is ignored and a WHERE clause selects rows + strictly after the given ``(sort_col, id)`` position in the active sort + direction. The cursor value must already be typed for the column + (datetime for time sorts, int for size, str for name); the caller decodes + the opaque cursor string and resolves to the typed value. + Returns (references, tag_map, total_count). """ base = ( @@ -297,9 +306,31 @@ def list_references_page( "size": Asset.size_bytes, } sort_col = sort_map.get(sort, AssetReference.created_at) - sort_exp = sort_col.desc() if order == "desc" else sort_col.asc() + descending = order == "desc" - base = base.order_by(sort_exp).limit(limit).offset(offset) + # Keyset WHERE: (sort_col, id) strictly less-than / greater-than the cursor. + # Equivalent to: sort_col v OR (sort_col = v AND id cursor_id). + if after_cursor_value is not None and after_cursor_id is not None: + if descending: + keyset = sa.or_( + sort_col < after_cursor_value, + sa.and_(sort_col == after_cursor_value, AssetReference.id < after_cursor_id), + ) + else: + keyset = sa.or_( + sort_col > after_cursor_value, + sa.and_(sort_col == after_cursor_value, AssetReference.id > after_cursor_id), + ) + base = base.where(keyset) + + # Secondary ORDER BY id (matching the primary direction) gives the keyset + # comparison a deterministic tiebreaker on duplicate sort_col values. + id_exp = AssetReference.id.desc() if descending else AssetReference.id.asc() + sort_exp = sort_col.desc() if descending else sort_col.asc() + + base = base.order_by(sort_exp, id_exp).limit(limit) + if after_cursor_id is None: + base = base.offset(offset) count_stmt = ( select(sa.func.count()) @@ -619,6 +650,7 @@ def upsert_reference( name: str, mtime_ns: int, owner_id: str = "", + loader_path: str | None = None, ) -> tuple[bool, bool]: """Upsert a reference by file_path. Returns (created, updated). @@ -628,6 +660,7 @@ def upsert_reference( vals = { "asset_id": asset_id, "file_path": file_path, + "loader_path": loader_path, "name": name, "owner_id": owner_id, "mtime_ns": int(mtime_ns), @@ -655,13 +688,14 @@ def upsert_reference( AssetReference.asset_id != asset_id, AssetReference.mtime_ns.is_(None), AssetReference.mtime_ns != int(mtime_ns), + AssetReference.loader_path.is_distinct_from(loader_path), AssetReference.is_missing == True, # noqa: E712 AssetReference.deleted_at.isnot(None), ) ) .values( - asset_id=asset_id, mtime_ns=int(mtime_ns), is_missing=False, - deleted_at=None, updated_at=now, + asset_id=asset_id, mtime_ns=int(mtime_ns), loader_path=loader_path, + is_missing=False, deleted_at=None, updated_at=now, ) ) res2 = session.execute(upd) diff --git a/app/assets/database/queries/tags.py b/app/assets/database/queries/tags.py index f4126dba8..148f34801 100644 --- a/app/assets/database/queries/tags.py +++ b/app/assets/database/queries/tags.py @@ -55,13 +55,11 @@ def validate_tags_exist(session: Session, tags: list[str]) -> None: raise ValueError(f"Unknown tags: {missing}") -def ensure_tags_exist( - session: Session, names: Iterable[str], tag_type: str = "user" -) -> None: +def ensure_tags_exist(session: Session, names: Iterable[str]) -> None: wanted = normalize_tags(list(names)) if not wanted: return - rows = [{"name": n, "tag_type": tag_type} for n in list(dict.fromkeys(wanted))] + rows = [{"name": n} for n in list(dict.fromkeys(wanted))] ins = ( sqlite.insert(Tag) .values(rows) @@ -97,7 +95,7 @@ def set_reference_tags( to_remove = [t for t in current if t not in desired] if to_add: - ensure_tags_exist(session, to_add, tag_type="user") + ensure_tags_exist(session, to_add) session.add_all( [ AssetReferenceTag( @@ -142,7 +140,7 @@ def add_tags_to_reference( return AddTagsResult(added=[], already_present=[], total_tags=total) if create_if_missing: - ensure_tags_exist(session, norm, tag_type="user") + ensure_tags_exist(session, norm) current = set(get_reference_tags(session, reference_id)) @@ -267,6 +265,8 @@ def list_tags_with_usage( order: str = "count_desc", owner_id: str = "", ) -> tuple[list[tuple[str, str, int]], int]: + prefix_filter = prefix.strip() if prefix else "" + counts_sq = ( select( AssetReferenceTag.tag_name.label("tag_name"), @@ -289,16 +289,14 @@ def list_tags_with_usage( q = ( select( Tag.name, - Tag.tag_type, func.coalesce(counts_sq.c.cnt, 0).label("count"), ) .select_from(Tag) .join(counts_sq, counts_sq.c.tag_name == Tag.name, isouter=True) ) - if prefix: - escaped, esc = escape_sql_like_string(prefix.strip().lower()) - q = q.where(Tag.name.like(escaped + "%", escape=esc)) + if prefix_filter: + q = q.where(func.substr(Tag.name, 1, len(prefix_filter)) == prefix_filter) if not include_zero: q = q.where(func.coalesce(counts_sq.c.cnt, 0) > 0) @@ -309,9 +307,8 @@ def list_tags_with_usage( q = q.order_by(func.coalesce(counts_sq.c.cnt, 0).desc(), Tag.name.asc()) total_q = select(func.count()).select_from(Tag) - if prefix: - escaped, esc = escape_sql_like_string(prefix.strip().lower()) - total_q = total_q.where(Tag.name.like(escaped + "%", escape=esc)) + if prefix_filter: + total_q = total_q.where(func.substr(Tag.name, 1, len(prefix_filter)) == prefix_filter) if not include_zero: visible_tags_sq = ( select(AssetReferenceTag.tag_name) @@ -331,7 +328,7 @@ def list_tags_with_usage( rows = (session.execute(q.limit(limit).offset(offset))).all() total = (session.execute(total_q)).scalar_one() - rows_norm = [(name, ttype, int(count or 0)) for (name, ttype, count) in rows] + rows_norm = [(name, int(count or 0)) for (name, count) in rows] return rows_norm, int(total or 0) diff --git a/app/assets/helpers.py b/app/assets/helpers.py index 3798f3933..87734d0dc 100644 --- a/app/assets/helpers.py +++ b/app/assets/helpers.py @@ -41,10 +41,10 @@ def get_utc_now() -> datetime: def normalize_tags(tags: list[str] | None) -> list[str]: """ Normalize a list of tags by: - - Stripping whitespace and converting to lowercase. - - Removing duplicates. + - Stripping whitespace. + - Removing exact duplicates while preserving order and case. """ - return list(dict.fromkeys(t.strip().lower() for t in (tags or []) if (t or "").strip())) + return list(dict.fromkeys(t.strip() for t in (tags or []) if (t or "").strip())) def validate_blake3_hash(s: str) -> str: diff --git a/app/assets/scanner.py b/app/assets/scanner.py index ebb6869af..42c4c1e9d 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -33,9 +33,10 @@ from app.assets.services.file_utils import ( verify_file_unchanged, ) from app.assets.services.hashing import HashCheckpoint, compute_blake3_hash +from app.assets.services.image_dimensions import extract_image_dimensions from app.assets.services.metadata_extract import extract_file_metadata from app.assets.services.path_utils import ( - compute_relative_filename, + compute_loader_path, get_comfy_models_folders, get_name_and_tags_from_asset_path, ) @@ -62,7 +63,7 @@ RootType = Literal["models", "input", "output"] def get_prefixes_for_root(root: RootType) -> list[str]: if root == "models": bases: list[str] = [] - for _bucket, paths in get_comfy_models_folders(): + for _bucket, paths, _exts in get_comfy_models_folders(): bases.extend(paths) return [os.path.abspath(p) for p in bases] if root == "input": @@ -80,7 +81,7 @@ def get_all_known_prefixes() -> list[str]: def collect_models_files() -> list[str]: out: list[str] = [] - for folder_name, bases in get_comfy_models_folders(): + for folder_name, bases, _exts in get_comfy_models_folders(): rel_files = folder_paths.get_filename_list(folder_name) or [] for rel_path in rel_files: if not all(is_visible(part) for part in Path(rel_path).parts): @@ -307,7 +308,7 @@ def build_asset_specs( if not stat_p.st_size: continue name, tags = get_name_and_tags_from_asset_path(abs_p) - rel_fname = compute_relative_filename(abs_p) + rel_fname = compute_loader_path(abs_p) # Extract metadata (tier 1: filesystem, tier 2: safetensors header) metadata = None @@ -354,7 +355,7 @@ def insert_asset_specs(specs: list[SeedAssetSpec], tag_pool: set[str]) -> int: return 0 with create_session() as sess: if tag_pool: - ensure_tags_exist(sess, tag_pool, tag_type="user") + ensure_tags_exist(sess, tag_pool) result = batch_insert_seed_assets(sess, specs=specs, owner_id="") sess.commit() return result.inserted_refs @@ -429,7 +430,7 @@ def enrich_asset( return new_level initial_mtime_ns = get_mtime_ns(stat_p) - rel_fname = compute_relative_filename(file_path) + rel_fname = compute_loader_path(file_path) mime_type: str | None = None metadata = None @@ -506,6 +507,10 @@ def enrich_asset( if extract_metadata and metadata: system_metadata = metadata.to_user_metadata() + if mime_type and mime_type.startswith("image/"): + dims = extract_image_dimensions(file_path, mime_type=mime_type) + if dims: + system_metadata.update(dims) set_reference_system_metadata(session, reference_id, system_metadata) if full_hash: diff --git a/app/assets/services/asset_management.py b/app/assets/services/asset_management.py index 5aefd9956..a4c8b5a75 100644 --- a/app/assets/services/asset_management.py +++ b/app/assets/services/asset_management.py @@ -1,8 +1,19 @@ import contextlib import mimetypes import os +from datetime import timezone from typing import Sequence +from app.assets.services.cursor import ( + CursorPayload, + InvalidCursorError, + decode_cursor, + decode_cursor_int, + decode_cursor_time, + encode_cursor, + encode_cursor_from_time, +) + from app.assets.database.models import Asset from app.assets.database.queries import ( @@ -27,7 +38,7 @@ from app.assets.database.queries import ( update_reference_updated_at, ) from app.assets.helpers import select_best_live_path -from app.assets.services.path_utils import compute_relative_filename +from app.assets.services.path_utils import compute_loader_path from app.assets.services.schemas import ( AssetData, AssetDetailResult, @@ -80,7 +91,7 @@ def update_asset_metadata( update_reference_name(session, reference_id=reference_id, name=name) touched = True - computed_filename = compute_relative_filename(ref.file_path) if ref.file_path else None + computed_filename = compute_loader_path(ref.file_path) if ref.file_path else None new_meta: dict | None = None if user_metadata is not None: @@ -149,6 +160,16 @@ def delete_asset_reference( owner_id: str, delete_content_if_orphan: bool = True, ) -> bool: + """Delete an asset reference. + + With ``delete_content_if_orphan=False`` (a soft delete), the reference is + hidden and the underlying content is preserved. With ``True``, the content + is also removed once it becomes orphaned. + + Note: the public DELETE /api/assets/{id} endpoint always soft-deletes + (passes ``False``); the orphan-reclamation path is intentionally + internal-only, retained for a future GC/admin caller. + """ with create_session() as session: if not delete_content_if_orphan: # Soft delete: mark the reference as deleted but keep everything @@ -242,6 +263,11 @@ def get_asset_by_hash(asset_hash: str) -> AssetData | None: return extract_asset_data(asset) +# Sort fields that support cursor pagination. `last_access_time` is not +# in this list — it falls back to offset/limit. +_CURSOR_SORT_FIELDS = ("created_at", "updated_at", "name", "size") + + def list_assets_page( owner_id: str = "", include_tags: Sequence[str] | None = None, @@ -252,7 +278,39 @@ def list_assets_page( offset: int = 0, sort: str = "created_at", order: str = "desc", + after: str | None = None, ) -> ListAssetsResult: + """List assets with optional cursor pagination. + + When ``after`` is supplied it overrides ``offset``. The cursor's sort field + must match ``sort`` and be in the cursor-supported allowlist; mismatches + raise InvalidCursorError so the handler can map to 400 INVALID_CURSOR. + """ + cursor_value: object | None = None + cursor_id: str | None = None + # Mint next_cursor on every page where the sort is cursor-supported, not + # only when the request itself arrived with a cursor. Otherwise a first + # request (no `after`) returns next_cursor=None and the client can never + # enter cursor mode. + mint_cursor = sort in _CURSOR_SORT_FIELDS + + if after is not None: + if sort not in _CURSOR_SORT_FIELDS: + raise InvalidCursorError( + f"cursor pagination is not supported for sort={sort!r}" + ) + payload = decode_cursor(after, _CURSOR_SORT_FIELDS, expected_order=order) + if payload.sort_field != sort: + raise InvalidCursorError( + f"cursor sort field {payload.sort_field!r} does not match request sort {sort!r}" + ) + cursor_value, cursor_id = _resolve_cursor_value(payload), payload.id + + # Over-fetch by one row so we can distinguish "exactly `limit` rows total + # remaining" from "more rows past this page" without a second query. Drop + # the sentinel before returning. + fetch_limit = limit + 1 if mint_cursor else limit + with create_session() as session: refs, tag_map, total = list_references_page( session, @@ -261,12 +319,22 @@ def list_assets_page( exclude_tags=exclude_tags, name_contains=name_contains, metadata_filter=metadata_filter, - limit=limit, + limit=fetch_limit, offset=offset, sort=sort, order=order, + after_cursor_value=cursor_value, + after_cursor_id=cursor_id, ) + next_cursor: str | None = None + if mint_cursor and len(refs) > limit: + # There's at least one more row past this page — mint a cursor from + # the last row of the page (i.e. index `limit - 1`, since we + # over-fetched), and drop the sentinel. + next_cursor = _encode_next_cursor(refs[limit - 1], sort, order) + refs = refs[:limit] + items: list[AssetSummaryData] = [] for ref in refs: items.append( @@ -277,7 +345,39 @@ def list_assets_page( ) ) - return ListAssetsResult(items=items, total=total) + return ListAssetsResult(items=items, total=total, next_cursor=next_cursor) + + +def _resolve_cursor_value(payload: CursorPayload) -> object: + """Map a decoded cursor payload to a column-typed Python value.""" + if payload.sort_field in ("created_at", "updated_at"): + # DB stores naive UTC; strip tzinfo so the comparison binds against a + # `TIMESTAMP WITHOUT TIME ZONE` column without an offset shift. + return decode_cursor_time(payload).replace(tzinfo=None) + if payload.sort_field == "size": + return decode_cursor_int(payload) + return payload.value # name, str-typed + + +def _encode_next_cursor(ref, sort: str, order: str) -> str | None: + """Mint a cursor pointing at *ref* for the given sort dimension. + + Returns None when the boundary row carries a NULL sort value (e.g. an asset + record whose size_bytes hasn't been backfilled). Continuing pagination + across a NULL boundary is undefined under keyset ordering — better to + truncate cleanly here than to mint a cursor that mis-positions. + """ + if sort == "name": + return encode_cursor("name", ref.name, ref.id, order=order) + if sort == "size": + if ref.asset is None or ref.asset.size_bytes is None: + return None + return encode_cursor("size", str(ref.asset.size_bytes), ref.id, order=order) + # created_at / updated_at — DB datetimes are naive UTC; attach tz before encoding. + value = ref.created_at if sort == "created_at" else ref.updated_at + if value is None: + return None + return encode_cursor_from_time(sort, value.replace(tzinfo=timezone.utc), ref.id, order=order) def resolve_hash_to_path( diff --git a/app/assets/services/bulk_ingest.py b/app/assets/services/bulk_ingest.py index 67aad838f..c98658bf1 100644 --- a/app/assets/services/bulk_ingest.py +++ b/app/assets/services/bulk_ingest.py @@ -56,6 +56,7 @@ class ReferenceRow(TypedDict): id: str asset_id: str file_path: str + loader_path: str | None mtime_ns: int owner_id: str name: str @@ -134,6 +135,14 @@ def batch_insert_seed_assets( for spec in specs: absolute_path = os.path.abspath(spec["abs_path"]) + existing_asset_id = path_to_asset_id.get(absolute_path) + if existing_asset_id is not None: + existing_tags = asset_id_to_ref_data[existing_asset_id]["tags"] + asset_id_to_ref_data[existing_asset_id]["tags"] = list( + dict.fromkeys([*existing_tags, *spec["tags"]]) + ) + continue + asset_id = str(uuid.uuid4()) reference_id = str(uuid.uuid4()) absolute_path_list.append(absolute_path) @@ -164,6 +173,8 @@ def batch_insert_seed_assets( "id": reference_id, "asset_id": asset_id, "file_path": absolute_path, + # spec["fname"] is compute_loader_path(abs_path) from build_asset_specs. + "loader_path": spec["fname"], "mtime_ns": spec["mtime_ns"], "owner_id": owner_id, "name": spec["info_name"], diff --git a/app/assets/services/cursor.py b/app/assets/services/cursor.py new file mode 100644 index 000000000..6c7791528 --- /dev/null +++ b/app/assets/services/cursor.py @@ -0,0 +1,213 @@ +"""Opaque keyset-pagination cursor for /api/assets. + +Payload JSON uses short keys to keep the encoded length small: + + {"s": , "v": , "id": , "o": } + +The `o` key binds the cursor to the sort direction it was minted under, +so replaying a `desc` cursor against an `asc` request fails with +``INVALID_CURSOR`` rather than silently walking the wrong direction. +`o` is mandatory on every payload — a cursor without it is rejected as +malformed. + +Encoding is base64url with no padding. Cursors are opaque tokens: the +payload format is internal to this server, and clients must treat a +cursor as a black box handed back via `next_cursor`. No byte-level +compatibility with any other implementation is required. + +Time values are serialized as Unix microseconds (UTC) — microsecond +precision is sufficient to round-trip the timestamps stored by the +database without rounding rows in the same millisecond bucket. +""" +from __future__ import annotations + +import base64 +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Iterable, Optional + + +class InvalidCursorError(ValueError): + """Raised on a malformed, oversized, or unsupported-sort-field cursor. + + Map to a 400 response with code ``INVALID_CURSOR`` at the handler. + """ + + +# Wire-format length caps. Cursors are user-controlled, so caps protect the +# decode path from oversized allocations and downstream SQL predicates from +# unbounded strings. +# +# MAX_CURSOR_VALUE_LENGTH is 512 to fit the `AssetReference.name` column max +# (`String(512)`) — otherwise a long-named asset would mint a cursor the same +# server then refuses on the next request. +# +# MAX_ENCODED_CURSOR_LENGTH is the decode-path guard, sized comfortably above +# the largest cursor the per-field caps can produce. Worst case is value + id +# at their caps with every character JSON-escaping to the six-byte `\uXXXX` +# form (control characters), which is ~5.2 KB once base64url-encoded. At 8192 +# the encoder can never mint a cursor that exceeds it, so a freshly minted +# cursor always decodes on the next request and there is no user-visible +# "cursor too long" failure. +MAX_ENCODED_CURSOR_LENGTH = 8192 +MAX_CURSOR_VALUE_LENGTH = 512 +MAX_CURSOR_ID_LENGTH = 128 + + +@dataclass(frozen=True) +class CursorPayload: + sort_field: str + value: str + id: str + order: str + + +_VALID_ORDERS = ("asc", "desc") + + +def encode_cursor(sort_field: str, value: str, id: str, order: str = "desc") -> str: + """Encode a cursor payload as a base64url (no-padding) string. + + `order` binds the cursor to the sort direction it was minted under so a + later request with a flipped `order` query parameter is rejected with + ``INVALID_CURSOR`` rather than silently walking the wrong direction. + """ + if order not in _VALID_ORDERS: + raise InvalidCursorError(f"order must be one of {_VALID_ORDERS}, got {order!r}") + # Symmetric input validation: the encoder must reject anything the + # decoder rejects, or the same server will mint cursors it then 400s on + # the next request. + if not id: + raise InvalidCursorError("id must be non-empty") + if len(id) > MAX_CURSOR_ID_LENGTH: + raise InvalidCursorError("id exceeds maximum length") + if len(value) > MAX_CURSOR_VALUE_LENGTH: + raise InvalidCursorError("value exceeds maximum length") + payload = {"s": sort_field, "v": value, "id": id, "o": order} + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False) + # No mint-time length guard is needed: the per-field caps above bound the + # encoded length well below MAX_ENCODED_CURSOR_LENGTH (see its definition), + # so the encoder can never produce a cursor the decode path would reject. + return base64.urlsafe_b64encode(raw.encode("utf-8")).rstrip(b"=").decode("ascii") + + +def encode_cursor_from_time(sort_field: str, t: datetime, id: str, order: str = "desc") -> str: + """Encode a time-typed cursor at Unix microsecond precision. + + Accepts an aware datetime (any timezone) and normalizes to UTC. Naive + datetimes are rejected so callers can't accidentally encode the local + wall-clock value of a UTC-stored timestamp. + """ + if t.tzinfo is None: + raise ValueError("encode_cursor_from_time requires an aware datetime") + micros = _datetime_to_unix_micros(t.astimezone(timezone.utc)) + return encode_cursor(sort_field, str(micros), id, order=order) + + +def decode_cursor( + cursor: str, + allowed_sort_fields: Iterable[str], + expected_order: str | None = None, +) -> CursorPayload: + """Parse an opaque cursor. + + ``allowed_sort_fields`` is the endpoint's accepted sort-field list — a + cursor carrying a field outside this set is rejected so a cursor minted + for one column can't be replayed against another (e.g. a ``created_at`` + timestamp string compared against a ``name`` column). + + ``expected_order`` (``"asc"``/``"desc"``), when supplied, must match the + payload's ``o`` field. ``o`` is required on every payload; a cursor + missing it is rejected as malformed. + + Passing no allowed fields rejects every cursor. + """ + if len(cursor) > MAX_ENCODED_CURSOR_LENGTH: + raise InvalidCursorError("cursor exceeds maximum length") + + try: + # urlsafe_b64decode requires correct padding; we strip on encode, so + # restore the trailing '=' pad here. + padding = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + padding) + except (ValueError, base64.binascii.Error) as e: + raise InvalidCursorError(f"encoding: {e}") from e + + try: + decoded = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + raise InvalidCursorError(f"payload: {e}") from e + + if not isinstance(decoded, dict): + raise InvalidCursorError("payload: expected object") + + sort_field = decoded.get("s") + value = decoded.get("v") + id = decoded.get("id") + order = decoded.get("o") + + if not isinstance(sort_field, str) or not isinstance(value, str) or not isinstance(id, str): + raise InvalidCursorError("payload: missing or non-string s/v/id") + + if id == "": + raise InvalidCursorError("missing id") + if len(id) > MAX_CURSOR_ID_LENGTH: + raise InvalidCursorError("id exceeds maximum length") + if len(value) > MAX_CURSOR_VALUE_LENGTH: + raise InvalidCursorError("value exceeds maximum length") + + if sort_field not in allowed_sort_fields: + raise InvalidCursorError(f"unsupported sort field {sort_field!r}") + + if not isinstance(order, str): + raise InvalidCursorError("missing or non-string o") + if order not in _VALID_ORDERS: + raise InvalidCursorError(f"unsupported order {order!r}") + if expected_order is not None and order != expected_order: + raise InvalidCursorError( + f"cursor order {order!r} does not match request order {expected_order!r}" + ) + + return CursorPayload(sort_field=sort_field, value=value, id=id, order=order) + + +def decode_cursor_time(payload: Optional[CursorPayload]) -> datetime: + """Parse a time-typed cursor value as Unix microseconds, returning UTC.""" + if payload is None: + raise InvalidCursorError("nil cursor payload") + try: + micros = int(payload.value) + except ValueError as e: + raise InvalidCursorError(f"value is not a valid timestamp: {e}") from e + try: + return _unix_micros_to_datetime(micros) + except (OverflowError, OSError, ValueError) as e: + # Crafted out-of-range microseconds (e.g. > datetime.MAX_YEAR) blow up + # in fromtimestamp / datetime construction. Map to 400, not 500. + raise InvalidCursorError(f"value is out of representable range: {e}") from e + + +def decode_cursor_int(payload: Optional[CursorPayload]) -> int: + """Parse a cursor value as a base-10 integer.""" + if payload is None: + raise InvalidCursorError("nil cursor payload") + try: + return int(payload.value) + except ValueError as e: + raise InvalidCursorError(f"value is not a valid integer: {e}") from e + + +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + + +def _datetime_to_unix_micros(t: datetime) -> int: + """Convert an aware UTC datetime to Unix microseconds (integer math).""" + delta = t - _EPOCH + return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds + + +def _unix_micros_to_datetime(micros: int) -> datetime: + """Convert Unix microseconds to a UTC datetime, preserving precision.""" + seconds, micro_remainder = divmod(micros, 1_000_000) + return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(microsecond=micro_remainder) diff --git a/app/assets/services/image_dimensions.py b/app/assets/services/image_dimensions.py new file mode 100644 index 000000000..ccd97399a --- /dev/null +++ b/app/assets/services/image_dimensions.py @@ -0,0 +1,63 @@ +"""Image dimension extraction for asset ingest. + +Reads only the image header via Pillow to capture width/height cheaply, +without a full pixel decode. Returns a metadata dict suitable for merging +into ``AssetReference.system_metadata``. +""" +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def extract_image_dimensions( + file_path: str, mime_type: str | None = None +) -> dict[str, Any] | None: + """Extract image dimensions for the file at ``file_path``. + + Args: + file_path: Absolute path to a file on disk. + mime_type: Optional MIME type hint. When provided and not prefixed + with ``image/``, extraction is skipped without touching the file. + + Returns: + ``{"kind": "image", "width": W, "height": H}`` when the file is a + recognizable image with positive dimensions, otherwise ``None``. + + The dict shape is intended to be merged into ``system_metadata`` so the + asset response surfaces ``metadata.kind`` plus dimension fields for image + assets. Forward-compatible: future media kinds (e.g. ``"video"`` with + duration/fps) can extend this shape without schema changes. + """ + if mime_type is not None and not mime_type.startswith("image/"): + return None + + try: + from PIL import Image, UnidentifiedImageError + except ImportError: + logger.debug( + "Pillow not available; skipping image dimension extraction for %s", + file_path, + ) + return None + + try: + with Image.open(file_path) as img: + width, height = img.size + except (OSError, UnidentifiedImageError, ValueError) as exc: + logger.debug( + "Failed to read image dimensions from %s: %s", file_path, exc + ) + return None + + if ( + not isinstance(width, int) + or not isinstance(height, int) + or width <= 0 + or height <= 0 + ): + return None + + return {"kind": "image", "width": width, "height": height} diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index f0b070517..1ffb3d634 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -17,9 +17,11 @@ from app.assets.database.queries import ( get_reference_by_file_path, get_reference_tags, get_or_create_reference, + list_references_by_asset_id, reference_exists, remove_missing_tag_for_asset_id, set_reference_metadata, + set_reference_system_metadata, set_reference_tags, update_asset_hash_and_mime, upsert_asset, @@ -29,9 +31,11 @@ from app.assets.database.queries import ( from app.assets.helpers import get_utc_now, normalize_tags from app.assets.services.bulk_ingest import batch_insert_seed_assets from app.assets.services.file_utils import get_size_and_mtime_ns +from app.assets.services.image_dimensions import extract_image_dimensions from app.assets.services.path_utils import ( - compute_relative_filename, + compute_loader_path, get_name_and_tags_from_asset_path, + get_path_derived_tags_from_path, resolve_destination_from_tags, validate_path_within_base, ) @@ -88,6 +92,7 @@ def _ingest_file_from_path( name=info_name or os.path.basename(locator), mtime_ns=mtime_ns, owner_id=owner_id, + loader_path=compute_loader_path(locator), ) # Get the reference we just created/updated @@ -98,17 +103,32 @@ def _ingest_file_from_path( if preview_id and ref.preview_id != preview_id: ref.preview_id = preview_id - norm = normalize_tags(list(tags)) - if norm: + try: + backend_tags = get_path_derived_tags_from_path(locator) + except ValueError: + backend_tags = [] + caller_tags = normalize_tags(tags) + backend_tags = normalize_tags(backend_tags) + all_tags = normalize_tags([*caller_tags, *backend_tags]) + if all_tags: if require_existing_tags: - validate_tags_exist(session, norm) - add_tags_to_reference( - session, - reference_id=reference_id, - tags=norm, - origin=tag_origin, - create_if_missing=not require_existing_tags, - ) + validate_tags_exist(session, all_tags) + if backend_tags: + add_tags_to_reference( + session, + reference_id=reference_id, + tags=backend_tags, + origin="automatic", + create_if_missing=not require_existing_tags, + ) + if caller_tags: + add_tags_to_reference( + session, + reference_id=reference_id, + tags=caller_tags, + origin=tag_origin, + create_if_missing=not require_existing_tags, + ) _update_metadata_with_filename( session, @@ -118,6 +138,14 @@ def _ingest_file_from_path( user_metadata=user_metadata, ) + _maybe_store_image_dimensions( + session, + reference_id=reference_id, + file_path=locator, + mime_type=mime_type, + current_system_metadata=ref.system_metadata, + ) + try: remove_missing_tag_for_asset_id(session, asset_id=asset.id) except Exception: @@ -217,7 +245,7 @@ def ingest_existing_file( "mtime_ns": mtime_ns, "info_name": name, "tags": tags, - "fname": os.path.basename(abs_path), + "fname": compute_loader_path(abs_path), "metadata": None, "hash": None, "mime_type": mime_type, @@ -277,7 +305,7 @@ def _register_existing_asset( return result new_meta = dict(user_metadata) - computed_filename = compute_relative_filename(ref.file_path) if ref.file_path else None + computed_filename = compute_loader_path(ref.file_path) if ref.file_path else None if computed_filename: new_meta["filename"] = computed_filename @@ -288,6 +316,13 @@ def _register_existing_asset( user_metadata=new_meta, ) + _backfill_image_dimensions_from_siblings( + session, + asset_id=asset.id, + new_reference_id=ref.id, + current_system_metadata=ref.system_metadata, + ) + if tags is not None: set_reference_tags( session, @@ -317,7 +352,7 @@ def _update_metadata_with_filename( current_metadata: dict | None, user_metadata: dict[str, Any], ) -> None: - computed_filename = compute_relative_filename(file_path) if file_path else None + computed_filename = compute_loader_path(file_path) if file_path else None current_meta = current_metadata or {} new_meta = dict(current_meta) @@ -334,6 +369,87 @@ def _update_metadata_with_filename( ) +_IMAGE_DIMENSION_KEYS = ("kind", "width", "height") + + +def _maybe_store_image_dimensions( + session: Session, + reference_id: str, + file_path: str, + mime_type: str | None, + current_system_metadata: dict | None, +) -> None: + """Populate ``kind``/``width``/``height`` on system_metadata for image refs. + + Non-image MIME types are a no-op. Pre-existing keys (e.g. enricher-written + safetensors metadata, download provenance) are preserved by merge. + """ + if not mime_type or not mime_type.startswith("image/"): + return + + dims = extract_image_dimensions(file_path, mime_type=mime_type) + if not dims: + return + + current = current_system_metadata or {} + merged = dict(current) + merged.update(dims) + if merged != current: + set_reference_system_metadata( + session, + reference_id=reference_id, + system_metadata=merged, + ) + + +def _backfill_image_dimensions_from_siblings( + session: Session, + asset_id: str, + new_reference_id: str, + current_system_metadata: dict | None, +) -> None: + """Copy image dimension keys from any sibling reference of the same asset. + + The from-hash path doesn't read the file bytes, so dimensions can't be + extracted there directly. When another reference of the same asset already + carries image dimensions, copy them onto the new reference so consumers + see consistent metadata regardless of how the asset was registered. + + Best-effort: missing siblings, non-image siblings, or absent dimension + keys leave the target reference unchanged. + """ + current = current_system_metadata or {} + if current.get("kind") == "image" and "width" in current and "height" in current: + return + + for sibling in list_references_by_asset_id(session, asset_id): + if sibling.id == new_reference_id: + continue + meta = sibling.system_metadata or {} + if meta.get("kind") != "image": + continue + width = meta.get("width") + height = meta.get("height") + if ( + type(width) is not int + or type(height) is not int + or width <= 0 + or height <= 0 + ): + continue + merged = dict(current) + merged["kind"] = "image" + merged["width"] = width + merged["height"] = height + if merged != current: + set_reference_system_metadata( + session, + reference_id=new_reference_id, + system_metadata=merged, + ) + return + + def _sanitize_filename(name: str | None, fallback: str) -> str: n = os.path.basename((name or "").strip() or fallback) return n if n else fallback @@ -375,6 +491,10 @@ def upload_from_temp_path( existing = get_asset_by_hash(session, asset_hash=asset_hash) if existing is not None: + # Once content is already known, duplicate byte uploads are treated as + # reference-only creation. Request tags are labels only here: do not + # require upload destination tags, do not move bytes, and do not + # synthesize path-derived classification or uploaded provenance. with contextlib.suppress(Exception): if temp_path and os.path.exists(temp_path): os.remove(temp_path) @@ -436,7 +556,7 @@ def upload_from_temp_path( owner_id=owner_id, preview_id=preview_id, user_metadata=user_metadata or {}, - tags=tags, + tags=[*(tags or []), "uploaded"], tag_origin="manual", require_existing_tags=False, ) @@ -470,15 +590,19 @@ def register_file_in_place( ) -> UploadResult: """Register an already-saved file in the asset database without moving it. - Tags are derived from the filesystem path (root category + subfolder names), - merged with any caller-provided tags, matching the behavior of the scanner. + This helper is used by upload paths that have already written bytes before + registering the file, so it records the same ``uploaded`` tag as the + multipart byte-upload path. + + Tags are derived from trusted filesystem classification and merged with any + caller-provided tags, matching the behavior of the scanner. If the path is not under a known root, only the caller-provided tags are used. """ try: _, path_tags = get_name_and_tags_from_asset_path(abs_path) except ValueError: path_tags = [] - merged_tags = normalize_tags([*path_tags, *tags]) + merged_tags = normalize_tags([*path_tags, *tags, "uploaded"]) try: digest, _ = hashing.compute_blake3_hash(abs_path) diff --git a/app/assets/services/path_utils.py b/app/assets/services/path_utils.py index 892140ffb..7c27c8878 100644 --- a/app/assets/services/path_utils.py +++ b/app/assets/services/path_utils.py @@ -3,59 +3,66 @@ from pathlib import Path from typing import Literal import folder_paths -from app.assets.helpers import normalize_tags -_NON_MODEL_FOLDER_NAMES = frozenset({"custom_nodes"}) +_NON_MODEL_FOLDER_NAMES = frozenset({"configs", "custom_nodes"}) +_KNOWN_SUBFOLDER_TAGS = frozenset({"3d", "pasted", "painter", "threed", "webcam"}) -def get_comfy_models_folders() -> list[tuple[str, list[str]]]: - """Build list of (folder_name, base_paths[]) for all model locations. +def get_comfy_models_folders() -> list[tuple[str, list[str], set[str]]]: + """Build list of (folder_name, base_paths[], extensions) for all model locations. Includes every category registered in folder_names_and_paths, regardless of whether its paths are under the main models_dir, - but excludes non-model entries like custom_nodes. + but excludes non-model entries like configs and custom_nodes. + + An empty extensions set means the category accepts any extension, + matching folder_paths.filter_files_extensions semantics. """ - targets: list[tuple[str, list[str]]] = [] + targets: list[tuple[str, list[str], set[str]]] = [] for name, values in folder_paths.folder_names_and_paths.items(): if name in _NON_MODEL_FOLDER_NAMES: continue - paths, _exts = values[0], values[1] + paths, exts = values[0], values[1] if paths: - targets.append((name, paths)) + targets.append((name, paths, set(exts))) return targets def resolve_destination_from_tags(tags: list[str]) -> tuple[str, list[str]]: - """Validates and maps tags -> (base_dir, subdirs_for_fs)""" - if not tags: - raise ValueError("tags must not be empty") - root = tags[0].lower() + """Validates and maps upload routing tags -> (base_dir, subdirs_for_fs). + + The request tags are only used to choose the write destination. Extra tags + remain labels; they do not become path components or trusted classification. + """ + destination_roles = [t for t in tags if t in {"input", "models", "output"}] + if len(destination_roles) != 1: + raise ValueError("uploads require exactly one destination role: input, models, or output") + + root = destination_roles[0] if root == "models": - if len(tags) < 2: - raise ValueError("at least two tags required for model asset") + model_type_tags = [t for t in tags if t.startswith("model_type:")] + if len(model_type_tags) != 1: + raise ValueError("models uploads require exactly one model_type: tag") + folder_name = model_type_tags[0].split(":", 1)[1] + if not folder_name: + raise ValueError("models uploads require exactly one model_type: tag") + model_folder_paths = { + name: paths for name, paths, _exts in get_comfy_models_folders() + } try: - bases = folder_paths.folder_names_and_paths[tags[1]][0] + bases = model_folder_paths[folder_name] except KeyError: - raise ValueError(f"unknown model category '{tags[1]}'") + raise ValueError(f"unknown model category '{folder_name}'") if not bases: - raise ValueError(f"no base path configured for category '{tags[1]}'") + raise ValueError(f"no base path configured for category '{folder_name}'") base_dir = os.path.abspath(bases[0]) - raw_subdirs = tags[2:] elif root == "input": base_dir = os.path.abspath(folder_paths.get_input_directory()) - raw_subdirs = tags[1:] - elif root == "output": - base_dir = os.path.abspath(folder_paths.get_output_directory()) - raw_subdirs = tags[1:] else: - raise ValueError(f"unknown root tag '{tags[0]}'; expected 'models', 'input', or 'output'") - _sep_chars = frozenset(("/", "\\", os.sep)) - for i in raw_subdirs: - if i in (".", "..") or _sep_chars & set(i): - raise ValueError("invalid path component in tags") + base_dir = os.path.abspath(folder_paths.get_output_directory()) - return base_dir, raw_subdirs if raw_subdirs else [] + return base_dir, [] def validate_path_within_base(candidate: str, base: str) -> None: @@ -65,14 +72,79 @@ def validate_path_within_base(candidate: str, base: str) -> None: raise ValueError("destination escapes base directory") -def compute_relative_filename(file_path: str) -> str | None: +def _compute_relative_path(child: str, parent: str) -> str: + rel = os.path.relpath(os.path.abspath(child), os.path.abspath(parent)) + if rel == ".": + return "" + return rel.replace(os.sep, "/") + + +def _is_relative_to(child: str, parent: str) -> bool: + return Path(os.path.abspath(child)).is_relative_to(os.path.abspath(parent)) + + +def compute_asset_response_paths(file_path: str) -> tuple[str, str | None] | None: + """Return (logical_path, display_name) for a file path. + + ``logical_path`` is the internal namespaced storage locator (e.g. + ``models/checkpoints/foo/bar.safetensors``); ``display_name`` is the + human-facing label below that namespace, served on Asset responses. These + are storage locators, not model-loader namespaces. Registered model-folder + membership is represented by backend tags such as + ``model_type:``; these paths only use known storage roots. """ - Return the model's path relative to the last well-known folder (the model category), - using forward slashes, eg: + fp_abs = os.path.abspath(file_path) + candidates: list[tuple[int, int, str, str]] = [] + + for order, (namespace, base) in enumerate( + ( + ("input", folder_paths.get_input_directory()), + ("output", folder_paths.get_output_directory()), + ("temp", folder_paths.get_temp_directory()), + ("models", getattr(folder_paths, "models_dir", "")), + ) + ): + if not base: + continue + base_abs = os.path.abspath(base) + if _is_relative_to(fp_abs, base_abs): + candidates.append((len(base_abs), -order, namespace, base_abs)) + + if not candidates: + return None + + _base_len, _order, namespace, base = max(candidates) + rel = _compute_relative_path(fp_abs, base) + public_path = f"{namespace}/{rel}" if rel else namespace + return public_path, rel or None + + +def compute_display_name(file_path: str) -> str | None: + """Return the asset's `display_name`, or None for unknown paths.""" + result = compute_asset_response_paths(file_path) + return result[1] if result else None + + +def compute_logical_path(file_path: str) -> str | None: + """Return the internal namespaced storage locator, or None for unknown paths.""" + result = compute_asset_response_paths(file_path) + return result[0] if result else None + + +def compute_loader_path(file_path: str) -> str | None: + """ + Return the asset's in-root loader path: the path relative to the last + well-known folder (the model category), using forward slashes, eg: /.../models/checkpoints/flux/123/flux.safetensors -> "flux/123/flux.safetensors" /.../models/text_encoders/clip_g.safetensors -> "clip_g.safetensors" - For non-model paths, returns None. + This is the value model loaders consume (the model category is dropped). It + is persisted as ``AssetReference.loader_path`` and served as the public + Asset response `loader_path` field. The human-facing `display_name` comes + from compute_asset_response_paths(). + + For input/output/temp paths the full path relative to that root is returned. + For paths outside any known root, returns None. """ try: root_category, rel_path = get_asset_category_and_relative_path(file_path) @@ -116,9 +188,10 @@ def get_asset_category_and_relative_path( def _compute_relative(child: str, parent: str) -> str: # Normalize relative path, stripping any leading ".." components # by anchoring to root (os.sep) then computing relpath back from it. - return os.path.relpath( + rel = os.path.relpath( os.path.join(os.sep, os.path.relpath(child, parent)), os.sep ) + return "" if rel == "." else rel.replace(os.sep, "/") # 1) input input_base = os.path.abspath(folder_paths.get_input_directory()) @@ -136,8 +209,14 @@ def get_asset_category_and_relative_path( return "temp", _compute_relative(fp_abs, temp_base) # 4) models (check deepest matching base to avoid ambiguity) + ext = os.path.splitext(fp_abs)[1].lower() best: tuple[int, str, str] | None = None # (base_len, bucket, rel_inside_bucket) - for bucket, bases in get_comfy_models_folders(): + for bucket, bases, extensions in get_comfy_models_folders(): + # A bucket only lists files within its extension set (empty set + # accepts any extension), so a bucket that cannot load the file + # must not contribute a loader path. + if extensions and ext not in extensions: + continue for b in bases: base_abs = os.path.abspath(b) if not _check_is_within(fp_abs, base_abs): @@ -149,25 +228,111 @@ def get_asset_category_and_relative_path( if best is not None: _, bucket, rel_inside = best combined = os.path.join(bucket, rel_inside) - return "models", os.path.relpath(os.path.join(os.sep, combined), os.sep) + normalized = os.path.relpath(os.path.join(os.sep, combined), os.sep) + return "models", normalized.replace(os.sep, "/") raise ValueError( f"Path is not within input, output, temp, or configured model bases: {file_path}" ) +def get_backend_system_tags_from_path(path: str) -> list[str]: + """Return trusted backend tags derived from current filesystem facts. + + The returned tags are only the backend-generated system tags: ``models``, + ``model_type:``, ``input``, ``output``, and ``temp``. Model + type tags are based on registered folder names, not path components. + + A ``model_type:`` tag is only emitted when the file's + extension is accepted by that folder's registered extension set, so + categories sharing a base directory tag only the files they can + actually load. Files under a model base whose extension matches no + category still get the ``models`` tag. + """ + fp_abs = os.path.abspath(path) + fp_path = Path(fp_abs) + tags: list[str] = [] + + def _add(tag: str) -> None: + if tag not in tags: + tags.append(tag) + + for role, base in ( + ("input", folder_paths.get_input_directory()), + ("output", folder_paths.get_output_directory()), + ("temp", folder_paths.get_temp_directory()), + ): + if fp_path.is_relative_to(os.path.abspath(base)): + _add(role) + + ext = os.path.splitext(fp_abs)[1].lower() + model_types: list[str] = [] + under_models_base = False + for folder_name, bases, extensions in get_comfy_models_folders(): + for base in bases: + if fp_path.is_relative_to(os.path.abspath(base)): + under_models_base = True + # Empty set accepts any extension, matching + # folder_paths.filter_files_extensions semantics. + if not extensions or ext in extensions: + model_types.append(folder_name) + break + + if under_models_base: + _add("models") + for folder_name in model_types: + _add(f"model_type:{folder_name}") + + if not tags: + raise ValueError( + f"Path is not within input, output, temp, or configured model bases: {path}" + ) + return tags + + +def get_known_subfolder_tags(subfolder: str | None) -> list[str]: + """Return tags for known UI/input subfolder names.""" + if subfolder in _KNOWN_SUBFOLDER_TAGS: + return [subfolder] + return [] + + +def get_known_input_subfolder_tags_from_path(path: str) -> list[str]: + """Return known input-layout tags for files in canonical input subfolders. + + These are compatibility tags for current UI-origin input directories such as + ``pasted`` and ``webcam``. They are intentionally narrow: only files directly + inside a known top-level input directory receive the matching tag. + """ + fp_abs = os.path.abspath(path) + input_base = os.path.abspath(folder_paths.get_input_directory()) + if not Path(fp_abs).is_relative_to(input_base): + return [] + + rel = os.path.relpath(fp_abs, input_base) + parts = Path(rel).parts + if len(parts) == 2: + return get_known_subfolder_tags(parts[0]) + return [] + + +def get_path_derived_tags_from_path(path: str) -> list[str]: + """Return all backend-derived tags for an asset path.""" + tags = get_backend_system_tags_from_path(path) + for tag in get_known_input_subfolder_tags_from_path(path): + if tag not in tags: + tags.append(tag) + return tags + + def get_name_and_tags_from_asset_path(file_path: str) -> tuple[str, list[str]]: """Return (name, tags) derived from a filesystem path. - name: base filename with extension - - tags: [root_category] + parent folder names in order + - tags: backend-derived tags from root/model classification and known input + subfolder layout conventions Raises: ValueError: path does not belong to any known root. """ - root_category, some_path = get_asset_category_and_relative_path(file_path) - p = Path(some_path) - parent_parts = [ - part for part in p.parent.parts if part not in (".", "..", p.anchor) - ] - return p.name, list(dict.fromkeys(normalize_tags([root_category, *parent_parts]))) + return Path(file_path).name, get_path_derived_tags_from_path(file_path) diff --git a/app/assets/services/schemas.py b/app/assets/services/schemas.py index 0eb128f58..0fda6871d 100644 --- a/app/assets/services/schemas.py +++ b/app/assets/services/schemas.py @@ -25,6 +25,7 @@ class ReferenceData: preview_id: str | None created_at: datetime updated_at: datetime + loader_path: str | None = None system_metadata: dict[str, Any] | None = None job_id: str | None = None last_access_time: datetime | None = None @@ -56,7 +57,6 @@ class IngestResult: class TagUsage(NamedTuple): name: str - tag_type: str count: int @@ -71,6 +71,7 @@ class AssetSummaryData: class ListAssetsResult: items: list[AssetSummaryData] total: int + next_cursor: str | None = None @dataclass(frozen=True) @@ -93,6 +94,7 @@ def extract_reference_data(ref: AssetReference) -> ReferenceData: id=ref.id, name=ref.name, file_path=ref.file_path, + loader_path=ref.loader_path, user_metadata=ref.user_metadata, preview_id=ref.preview_id, system_metadata=ref.system_metadata, diff --git a/app/assets/services/tagging.py b/app/assets/services/tagging.py index 37b612753..5fa39d26a 100644 --- a/app/assets/services/tagging.py +++ b/app/assets/services/tagging.py @@ -75,7 +75,7 @@ def list_tags( owner_id=owner_id, ) - return [TagUsage(name, tag_type, count) for name, tag_type, count in rows], total + return [TagUsage(name, count) for name, count in rows], total def list_tag_histogram( diff --git a/app/model_manager.py b/app/model_manager.py index 8f6e34b33..5928781ca 100644 --- a/app/model_manager.py +++ b/app/model_manager.py @@ -35,7 +35,11 @@ class ModelFileManager: for folder in model_types: if folder in folder_black_list: continue - output_folders.append({"name": folder, "folders": folder_paths.get_folder_paths(folder)}) + output_folders.append({ + "name": folder, + "folders": folder_paths.get_folder_paths(folder), + "extensions": sorted(folder_paths.folder_names_and_paths[folder][1]), + }) return web.json_response(output_folders) # NOTE: This is an experiment to replace `/models/{folder}` @@ -50,21 +54,45 @@ class ModelFileManager: @routes.get("/experiment/models/preview/{folder}/{path_index}/{filename:.*}") async def get_model_preview(request): folder_name = request.match_info.get("folder", None) - path_index = int(request.match_info.get("path_index", None)) filename = request.match_info.get("filename", None) if folder_name not in folder_paths.folder_names_and_paths: return web.Response(status=404) + # The "{filename:.*}" capture also matches the empty string, which + # would resolve to the folder itself; reject it explicitly. + if not filename: + return web.Response(status=400) + + try: + path_index = int(request.match_info.get("path_index", None)) + except (TypeError, ValueError): + return web.Response(status=400) + folders = folder_paths.folder_names_and_paths[folder_name] + if path_index < 0 or path_index >= len(folders[0]): + return web.Response(status=404) folder = folders[0][path_index] - full_filename = os.path.join(folder, filename) + full_filename = os.path.normpath(os.path.join(folder, filename)) + + # Prevent path traversal: the requested file must stay within the + # configured model folder. `filename` is an unrestricted ".*" capture, + # so values like "../../../../etc/passwd" would otherwise escape it. + if not folder_paths.is_within_directory(folder, full_filename): + return web.Response(status=403) previews = self.get_model_previews(full_filename) default_preview = previews[0] if len(previews) > 0 else None if default_preview is None or (isinstance(default_preview, str) and not os.path.isfile(default_preview)): return web.Response(status=404) + # The preview is selected by a glob inside get_model_previews, so a + # companion file (e.g. "model.preview.png") could itself be a symlink + # resolving outside the model folder. Re-validate the file actually + # opened: is_within_directory realpaths it, catching symlink escape. + if isinstance(default_preview, str) and not folder_paths.is_within_directory(folder, default_preview): + return web.Response(status=403) + try: with Image.open(default_preview) as img: img_bytes = BytesIO() diff --git a/app/user_manager.py b/app/user_manager.py index 7b11e381c..de261ad39 100644 --- a/app/user_manager.py +++ b/app/user_manager.py @@ -6,6 +6,7 @@ import glob import shutil import logging import tempfile +import mimetypes from aiohttp import web from urllib import parse from comfy.cli_args import args @@ -336,7 +337,20 @@ class UserManager(): if not isinstance(path, str): return path - return web.FileResponse(path) + # User data files are arbitrary user-supplied content and are never + # meant to render inline. Disable MIME sniffing and force a download + # so uploaded markup/scripts can't execute in the app origin (stored + # XSS). Content-Disposition: attachment is the load-bearing guard; + # the content-type override and nosniff are defence in depth. + content_type = mimetypes.guess_type(path)[0] or 'application/octet-stream' + if folder_paths.is_dangerous_content_type(content_type): + content_type = 'application/octet-stream' + + return web.FileResponse(path, headers={ + "Content-Type": content_type, + "X-Content-Type-Options": "nosniff", + "Content-Disposition": "attachment", + }) @routes.post("/userdata/{file}") async def post_userdata(request): diff --git a/blueprints/Character Replacement (SCAIL-2 Base).json b/blueprints/Character Replacement (SCAIL-2 Base).json new file mode 100644 index 000000000..61803df65 --- /dev/null +++ b/blueprints/Character Replacement (SCAIL-2 Base).json @@ -0,0 +1,4191 @@ +{ + "revision": 0, + "last_node_id": 410, + "last_link_id": 0, + "nodes": [ + { + "id": 410, + "type": "35331397-69fb-40ad-b99a-7f17b1a53017", + "pos": [ + 2450, + 5670 + ], + "size": [ + 490, + 1120 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "label": "pose_video", + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "label": "reference_image", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": null + }, + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "segment_index", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "replace_mode", + "name": "value_2", + "type": "BOOLEAN", + "widget": { + "name": "value_2" + }, + "link": null + }, + { + "label": "width", + "name": "value_3", + "type": "INT", + "widget": { + "name": "value_3" + }, + "link": null + }, + { + "label": "height", + "name": "value_4", + "type": "INT", + "widget": { + "name": "value_4" + }, + "link": null + }, + { + "label": "frame_count", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + }, + { + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": null + }, + { + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": null + }, + { + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": null + }, + { + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": null + }, + { + "label": "turbo_mode", + "name": "value_5", + "type": "BOOLEAN", + "widget": { + "name": "value_5" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "distill_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "label": "dpo_lora", + "name": "lora_name_1", + "type": "COMBO", + "widget": { + "name": "lora_name_1" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "clip_vision", + "name": "clip_name_1", + "type": "COMBO", + "widget": { + "name": "clip_name_1" + }, + "link": null + }, + { + "label": "sam3_video_object", + "name": "text_1", + "type": "STRING", + "widget": { + "name": "text_1" + }, + "link": null + }, + { + "label": "sam3_image_object", + "name": "text_2", + "type": "STRING", + "widget": { + "name": "text_2" + }, + "link": null + }, + { + "label": "sam3_model", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "405", + "text" + ], + [ + "391", + "value" + ], + [ + "398", + "value" + ], + [ + "387", + "value" + ], + [ + "388", + "value" + ], + [ + "386", + "length" + ], + [ + "406", + "previous_frame_count" + ], + [ + "406", + "pose_strength" + ], + [ + "406", + "pose_start" + ], + [ + "406", + "pose_end" + ], + [ + "402", + "value" + ], + [ + "374", + "unet_name" + ], + [ + "367", + "lora_name" + ], + [ + "408", + "lora_name" + ], + [ + "372", + "clip_name" + ], + [ + "368", + "vae_name" + ], + [ + "373", + "clip_name" + ], + [ + "393", + "text" + ], + [ + "407", + "text" + ], + [ + "394", + "ckpt_name" + ], + [ + "378", + "noise_seed" + ], + [ + "399", + "$$canvas-image-preview" + ], + [ + "400", + "$$canvas-image-preview" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Character Replacement (SCAIL-2 Base)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "35331397-69fb-40ad-b99a-7f17b1a53017", + "version": 1, + "state": { + "lastGroupId": 17, + "lastNodeId": 410, + "lastLinkId": 570, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Character Replacement (SCAIL-2 Base)", + "inputNode": { + "id": -10, + "bounding": [ + -440, + 4190, + 173.015625, + 508 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 3980, + 3870, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "e9aedfaf-1991-4a75-ad9b-8c3a279b0488", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 409 + ], + "localized_name": "video", + "label": "pose_video", + "pos": [ + -290.984375, + 4214 + ] + }, + { + "id": "0badb26e-9abd-4e9c-b221-aab1237b8773", + "name": "images", + "type": "IMAGE", + "linkIds": [ + 469, + 408, + 473 + ], + "localized_name": "images", + "label": "reference_image", + "pos": [ + -290.984375, + 4234 + ] + }, + { + "id": "098f15aa-a066-422e-b491-eaf140bafb2c", + "name": "text", + "type": "STRING", + "linkIds": [ + 499 + ], + "label": "prompt", + "pos": [ + -290.984375, + 4254 + ] + }, + { + "id": "861ea850-8329-491b-9413-2588cdd77bbd", + "name": "value", + "type": "INT", + "linkIds": [ + 502 + ], + "label": "segment_index", + "pos": [ + -290.984375, + 4274 + ] + }, + { + "id": "dcf3ebf8-1263-458a-966b-9e77a5ad704a", + "name": "value_2", + "type": "BOOLEAN", + "linkIds": [ + 504 + ], + "label": "replace_mode", + "pos": [ + -290.984375, + 4294 + ] + }, + { + "id": "dbee10ce-bb98-4733-88a0-d4cd5b13c691", + "name": "value_3", + "type": "INT", + "linkIds": [ + 505 + ], + "label": "width", + "pos": [ + -290.984375, + 4314 + ] + }, + { + "id": "363e3ac7-0096-4480-b875-1d6ff1b8a2d9", + "name": "value_4", + "type": "INT", + "linkIds": [ + 506 + ], + "label": "height", + "pos": [ + -290.984375, + 4334 + ] + }, + { + "id": "4017e1ed-96cd-4c1c-9949-dc1a8c30386a", + "name": "length", + "type": "INT", + "linkIds": [ + 522, + 541 + ], + "label": "frame_count", + "pos": [ + -290.984375, + 4354 + ] + }, + { + "id": "617809fd-9647-4067-8f44-10d8acc8af82", + "name": "previous_frame_count", + "type": "INT", + "linkIds": [ + 542, + 543 + ], + "pos": [ + -290.984375, + 4374 + ] + }, + { + "id": "36ab49f6-528f-4301-a37a-0108aa7f5bec", + "name": "pose_strength", + "type": "FLOAT", + "linkIds": [ + 547 + ], + "pos": [ + -290.984375, + 4394 + ] + }, + { + "id": "24c2e93f-7bbc-4c6d-898e-36b440b8469b", + "name": "pose_start", + "type": "FLOAT", + "linkIds": [ + 548 + ], + "pos": [ + -290.984375, + 4414 + ] + }, + { + "id": "d1dbe632-bb2a-4c30-a7ea-5082f21dc686", + "name": "pose_end", + "type": "FLOAT", + "linkIds": [ + 549 + ], + "pos": [ + -290.984375, + 4434 + ] + }, + { + "id": "ddf07ba4-2837-40ef-925f-5996ea436334", + "name": "value_5", + "type": "BOOLEAN", + "linkIds": [ + 507 + ], + "label": "turbo_mode", + "pos": [ + -290.984375, + 4454 + ] + }, + { + "id": "ee8c796d-b326-40c6-9f9d-65f564053974", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 508 + ], + "pos": [ + -290.984375, + 4474 + ] + }, + { + "id": "6ccae991-51d7-4c8e-adc8-c7f6377d681b", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 509 + ], + "label": "distill_lora", + "pos": [ + -290.984375, + 4494 + ] + }, + { + "id": "30772913-ecf9-44f4-a638-2824404b24fb", + "name": "lora_name_1", + "type": "COMBO", + "linkIds": [ + 565 + ], + "label": "dpo_lora", + "pos": [ + -290.984375, + 4514 + ] + }, + { + "id": "2e9c2347-93fe-462f-b5ef-51613fb52c85", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 510 + ], + "pos": [ + -290.984375, + 4534 + ] + }, + { + "id": "bc1822c0-7484-47b7-8d59-cf94788290f7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 511 + ], + "pos": [ + -290.984375, + 4554 + ] + }, + { + "id": "bbf2ee19-58b3-46f4-af6a-cd1557d60c8d", + "name": "clip_name_1", + "type": "COMBO", + "linkIds": [ + 512 + ], + "label": "clip_vision", + "pos": [ + -290.984375, + 4574 + ] + }, + { + "id": "556415bd-8ef3-4cfa-b182-0d7e3e4cd572", + "name": "text_1", + "type": "STRING", + "linkIds": [ + 513 + ], + "label": "sam3_video_object", + "pos": [ + -290.984375, + 4594 + ] + }, + { + "id": "c577930c-85a3-4e92-ac44-8ed639937217", + "name": "text_2", + "type": "STRING", + "linkIds": [ + 514 + ], + "label": "sam3_image_object", + "pos": [ + -290.984375, + 4614 + ] + }, + { + "id": "8f542c5e-da91-4dad-8ff8-4e81a42d5140", + "name": "ckpt_name", + "type": "COMBO", + "linkIds": [ + 515 + ], + "label": "sam3_model", + "pos": [ + -290.984375, + 4634 + ] + }, + { + "id": "3064d232-69a3-43b1-b350-05d8ca0e7c9f", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 558 + ], + "pos": [ + -290.984375, + 4654 + ] + } + ], + "outputs": [ + { + "id": "7ae88834-7553-45ec-a4e8-bab7d5276b45", + "name": "output", + "type": "IMAGE", + "linkIds": [ + 544 + ], + "localized_name": "output", + "pos": [ + 4004, + 3894 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 367, + "type": "LoraLoaderModelOnly", + "pos": [ + 300, + 3590 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 563 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 509 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 398, + 425 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 0.8 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 368, + "type": "VAELoader", + "pos": [ + 300, + 4020 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 511 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 406, + 407 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 369, + "type": "ResizeImageMaskNode", + "pos": [ + 1280, + 4860 + ], + "size": [ + 270, + 160 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "input", + "name": "input", + "type": "IMAGE,MASK", + "link": 491 + }, + { + "localized_name": "resize_type", + "name": "resize_type", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "resize_type" + }, + "link": null + }, + { + "localized_name": "width", + "name": "resize_type.width", + "type": "INT", + "widget": { + "name": "resize_type.width" + }, + "link": 444 + }, + { + "localized_name": "height", + "name": "resize_type.height", + "type": "INT", + "widget": { + "name": "resize_type.height" + }, + "link": 446 + }, + { + "localized_name": "crop", + "name": "resize_type.crop", + "type": "COMBO", + "widget": { + "name": "resize_type.crop" + }, + "link": null + }, + { + "localized_name": "scale_method", + "name": "scale_method", + "type": "COMBO", + "widget": { + "name": "scale_method" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "resized", + "name": "resized", + "type": "*", + "links": [ + 420, + 492, + 493 + ] + } + ], + "properties": { + "Node name for S&R": "ResizeImageMaskNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "scale dimensions", + 512, + 512, + "center", + "area" + ] + }, + { + "id": 370, + "type": "GetVideoComponents", + "pos": [ + 270, + 4500 + ], + "size": [ + 230, + 90 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 409 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 490 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 371, + "type": "GetImageSize", + "pos": [ + 1640, + 4500 + ], + "size": [ + 240, + 190 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 492 + } + ], + "outputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "links": [ + 414 + ] + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "links": [ + 415 + ] + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "links": [ + 416 + ] + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 372, + "type": "CLIPLoader", + "pos": [ + 300, + 3790 + ], + "size": [ + 590, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 510 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 7, + 8 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "directory": "text_encoders" + } + ] + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 373, + "type": "CLIPVisionLoader", + "pos": [ + 300, + 4230 + ], + "size": [ + 590, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 512 + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 196 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "clip_vision_h.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/clip_vision/clip_vision_h.safetensors", + "directory": "clip_vision" + } + ] + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ] + }, + { + "id": 374, + "type": "UNETLoader", + "pos": [ + 300, + 3190 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 508 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 562 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_14B_SCAIL_2_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/diffusion_models/wan2.1_14B_SCAIL_2_fp16.safetensors", + "directory": "diffusion_models" + } + ] + }, + "widgets_values": [ + "wan2.1_14B_SCAIL_2_fp16.safetensors", + "default" + ] + }, + { + "id": 375, + "type": "ModelSamplingSD3", + "pos": [ + 2760, + 3370 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 424 + }, + { + "localized_name": "shift", + "name": "shift", + "type": "FLOAT", + "widget": { + "name": "shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 417 + ] + } + ], + "properties": { + "Node name for S&R": "ModelSamplingSD3", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 376, + "type": "KSamplerSelect", + "pos": [ + 2770, + 3520 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 419 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 377, + "type": "BasicScheduler", + "pos": [ + 2770, + 3670 + ], + "size": [ + 260, + 170 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 398 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 428 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 418 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 378, + "type": "SamplerCustom", + "pos": [ + 3080, + 3370 + ], + "size": [ + 270, + 670 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 417 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 333 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 334 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 419 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 418 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 335 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 558 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 431 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [ + 124 + ] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true, + 1, + "fixed", + 1 + ] + }, + { + "id": 379, + "type": "PrimitiveInt", + "pos": [ + 2060, + 3750 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 427 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 380, + "type": "PrimitiveFloat", + "pos": [ + 2060, + 3920 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 430 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 381, + "type": "PrimitiveInt", + "pos": [ + 2060, + 3380 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 426 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 382, + "type": "PrimitiveFloat", + "pos": [ + 2060, + 3530 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 429 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 383, + "type": "ComfySwitchNode", + "pos": [ + 2400, + 3360 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 564 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 425 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 432 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 424 + ] + } + ], + "title": "Switch (Model)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 384, + "type": "ComfySwitchNode", + "pos": [ + 2400, + 3550 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 426 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 427 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 433 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 428 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 385, + "type": "ComfySwitchNode", + "pos": [ + 2400, + 3730 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 429 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 430 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 434 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 431 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 386, + "type": "ImageFromBatch", + "pos": [ + 880, + 4490 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 490 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": 450 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 522 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 491 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0, + 81 + ] + }, + { + "id": 387, + "type": "PrimitiveInt", + "pos": [ + 250, + 4820 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 505 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Int (Width)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 896, + "fixed" + ] + }, + { + "id": 388, + "type": "PrimitiveInt", + "pos": [ + 250, + 5000 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 506 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 445 + ] + } + ], + "title": "Int (Height)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + "fixed" + ] + }, + { + "id": 389, + "type": "ComfyMathExpression", + "pos": [ + 690, + 4850 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 441 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 444 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 390, + "type": "ComfyMathExpression", + "pos": [ + 690, + 5030 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 445 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 446 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 391, + "type": "PrimitiveInt", + "pos": [ + 260, + 4570 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 502 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 447 + ] + } + ], + "title": "Int (segment index)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1, + "fixed" + ] + }, + { + "id": 392, + "type": "ComfyMathExpression", + "pos": [ + 580, + 4590 + ], + "size": [ + 230, + 220 + ], + "flags": { + "collapsed": false + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 447 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 541 + }, + { + "label": "c", + "localized_name": "values.c", + "name": "values.c", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 543 + }, + { + "label": "d", + "localized_name": "values.d", + "name": "values.d", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 450 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(b - c) * (a - 1)" + ] + }, + { + "id": 393, + "type": "CLIPTextEncode", + "pos": [ + 660, + 5360 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 454 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 513 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 461 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 394, + "type": "CheckpointLoaderSimple", + "pos": [ + 270, + 5340 + ], + "size": [ + 330, + 160 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "ckpt_name", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": 515 + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 458, + 463 + ] + }, + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 454, + 489 + ] + }, + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [] + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "models": [ + { + "name": "sam3.1_multiplex_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/sam3.1/resolve/main/checkpoints/sam3.1_multiplex_fp16.safetensors", + "directory": "checkpoints" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "sam3.1_multiplex_fp16.safetensors" + ] + }, + { + "id": 395, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5320 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 493 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 458 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 461 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 460 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 396, + "type": "SCAIL2ColoredMask", + "pos": [ + 1550, + 5470 + ], + "size": [ + 370, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "driving_track_data", + "name": "driving_track_data", + "type": "SAM3_TRACK_DATA", + "link": 460 + }, + { + "localized_name": "ref_track_data", + "name": "ref_track_data", + "shape": 7, + "type": "SAM3_TRACK_DATA", + "link": 464 + }, + { + "localized_name": "object_indices", + "name": "object_indices", + "type": "STRING", + "widget": { + "name": "object_indices" + }, + "link": null + }, + { + "localized_name": "sort_by", + "name": "sort_by", + "type": "COMBO", + "widget": { + "name": "sort_by" + }, + "link": null + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 476 + } + ], + "outputs": [ + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "type": "IMAGE", + "links": [ + 466, + 467 + ] + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "type": "IMAGE", + "links": [ + 465, + 472 + ] + } + ], + "properties": { + "Node name for S&R": "SCAIL2ColoredMask", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "left_to_right", + true + ] + }, + { + "id": 397, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5620 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 469 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 463 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 488 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 464 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 398, + "type": "PrimitiveBoolean", + "pos": [ + 1660, + 4200 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 504 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 475, + 476 + ] + } + ], + "title": "Boolean （Replace Mode）", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 399, + "type": "PreviewImage", + "pos": [ + 2040, + 4470 + ], + "size": [ + 350, + 1190 + ], + "flags": {}, + "order": 32, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 466 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 400, + "type": "PreviewImage", + "pos": [ + 2440, + 4470 + ], + "size": [ + 230, + 310 + ], + "flags": {}, + "order": 33, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 465 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 401, + "type": "VAEDecode", + "pos": [ + 3080, + 4120 + ], + "size": [ + 270, + 100 + ], + "flags": { + "collapsed": false + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 124 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 407 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 544 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 402, + "type": "PrimitiveBoolean", + "pos": [ + 2410, + 3940 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 507 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 432, + 433, + 434 + ] + } + ], + "title": "Boolean (Enable Distill LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 403, + "type": "CLIPVisionEncode", + "pos": [ + 1200, + 4260 + ], + "size": [ + 230, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 196 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 408 + }, + { + "localized_name": "crop", + "name": "crop", + "type": "COMBO", + "widget": { + "name": "crop" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION_OUTPUT", + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "links": [ + 404 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 404, + "type": "CLIPTextEncode", + "pos": [ + 1000, + 3970 + ], + "size": [ + 520, + 210 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 326 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 405, + "type": "CLIPTextEncode", + "pos": [ + 990, + 3170 + ], + "size": [ + 550, + 750 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 499 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 325 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "A young woman with dark hair tied in a neat high bun, with a few loose strands framing her face, is dancing outdoors on a sunny coastal hillside. She has a normal-sized head and a slim face, with no hat, no headwear, and no oversized hair volume. She wears a fitted black long-sleeve crop top with a shoulder cutout, extremely baggy black cargo pants with straps and pockets, and chunky black combat boots. She performs energetic dance moves with one leg lifted and arms extended, moving naturally in front of a large tree, a small white stone house with a terracotta roof, and a bright blue sea under a clear sky with light clouds." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 406, + "type": "WanSCAILToVideo", + "pos": [ + 1640, + 3400 + ], + "size": [ + 310, + 580 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 325 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 326 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 406 + }, + { + "localized_name": "pose_video", + "name": "pose_video", + "shape": 7, + "type": "IMAGE", + "link": 420 + }, + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "shape": 7, + "type": "IMAGE", + "link": 467 + }, + { + "localized_name": "reference_image", + "name": "reference_image", + "shape": 7, + "type": "IMAGE", + "link": 473 + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "shape": 7, + "type": "IMAGE", + "link": 472 + }, + { + "localized_name": "clip_vision_output", + "name": "clip_vision_output", + "shape": 7, + "type": "CLIP_VISION_OUTPUT", + "link": 404 + }, + { + "localized_name": "previous_frames", + "name": "previous_frames", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 414 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 415 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 416 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "pose_strength", + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": 547 + }, + { + "localized_name": "pose_start", + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": 548 + }, + { + "localized_name": "pose_end", + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": 549 + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "widget": { + "name": "video_frame_offset" + }, + "link": null + }, + { + "localized_name": "previous_frame_count", + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": 542 + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 475 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 333 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 334 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 335 + ] + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanSCAILToVideo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + 896, + 65, + 1, + 1, + 0, + 1, + 0, + 5, + true + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 407, + "type": "CLIPTextEncode", + "pos": [ + 670, + 5690 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 489 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 514 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 488 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 408, + "type": "LoraLoaderModelOnly", + "pos": [ + 310, + 3390 + ], + "size": [ + 580, + 140 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 562 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 565 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 563, + 564 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/loras/wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + 1 + ] + } + ], + "groups": [ + { + "id": 3, + "title": "Models", + "bounding": [ + 240, + 3100, + 710, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 980, + 3100, + 570, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Sampling", + "bounding": [ + 2730, + 3100, + 660, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "SCAIL Conditioning", + "bounding": [ + 1580, + 3100, + 420, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 9, + "title": "Switch Settings", + "bounding": [ + 2030, + 3100, + 670, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Original Settings", + "bounding": [ + 2050, + 3320, + 300, + 340 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Distill LoRA Settings", + "bounding": [ + 2050, + 3680, + 300, + 370 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 10, + "title": "Create Mask", + "bounding": [ + 240, + 5230, + 1760, + 670 + ], + "flags": {} + }, + { + "id": 11, + "title": "Resize Images", + "bounding": [ + 240, + 4750, + 1760, + 450 + ], + "flags": {} + }, + { + "id": 12, + "title": "Trim Frames from Video", + "bounding": [ + 240, + 4410, + 1760, + 310 + ], + "flags": {} + }, + { + "id": 13, + "title": "Preview Masks", + "bounding": [ + 2030, + 4410, + 660, + 1490 + ], + "flags": {} + } + ], + "links": [ + { + "id": 491, + "origin_id": 386, + "origin_slot": 0, + "target_id": 369, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 444, + "origin_id": 389, + "origin_slot": 1, + "target_id": 369, + "target_slot": 2, + "type": "INT" + }, + { + "id": 446, + "origin_id": 390, + "origin_slot": 1, + "target_id": 369, + "target_slot": 3, + "type": "INT" + }, + { + "id": 492, + "origin_id": 369, + "origin_slot": 0, + "target_id": 371, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 424, + "origin_id": 383, + "origin_slot": 0, + "target_id": 375, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 398, + "origin_id": 367, + "origin_slot": 0, + "target_id": 377, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 428, + "origin_id": 384, + "origin_slot": 0, + "target_id": 377, + "target_slot": 2, + "type": "INT" + }, + { + "id": 417, + "origin_id": 375, + "origin_slot": 0, + "target_id": 378, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 333, + "origin_id": 406, + "origin_slot": 0, + "target_id": 378, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 334, + "origin_id": 406, + "origin_slot": 1, + "target_id": 378, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 419, + "origin_id": 376, + "origin_slot": 0, + "target_id": 378, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 418, + "origin_id": 377, + "origin_slot": 0, + "target_id": 378, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 335, + "origin_id": 406, + "origin_slot": 2, + "target_id": 378, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 431, + "origin_id": 385, + "origin_slot": 0, + "target_id": 378, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 425, + "origin_id": 367, + "origin_slot": 0, + "target_id": 383, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 432, + "origin_id": 402, + "origin_slot": 0, + "target_id": 383, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 426, + "origin_id": 381, + "origin_slot": 0, + "target_id": 384, + "target_slot": 0, + "type": "INT" + }, + { + "id": 427, + "origin_id": 379, + "origin_slot": 0, + "target_id": 384, + "target_slot": 1, + "type": "INT" + }, + { + "id": 433, + "origin_id": 402, + "origin_slot": 0, + "target_id": 384, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 429, + "origin_id": 382, + "origin_slot": 0, + "target_id": 385, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 430, + "origin_id": 380, + "origin_slot": 0, + "target_id": 385, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 434, + "origin_id": 402, + "origin_slot": 0, + "target_id": 385, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 490, + "origin_id": 370, + "origin_slot": 0, + "target_id": 386, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 450, + "origin_id": 392, + "origin_slot": 1, + "target_id": 386, + "target_slot": 1, + "type": "INT" + }, + { + "id": 441, + "origin_id": 387, + "origin_slot": 0, + "target_id": 389, + "target_slot": 0, + "type": "INT" + }, + { + "id": 445, + "origin_id": 388, + "origin_slot": 0, + "target_id": 390, + "target_slot": 0, + "type": "INT" + }, + { + "id": 447, + "origin_id": 391, + "origin_slot": 0, + "target_id": 392, + "target_slot": 0, + "type": "INT" + }, + { + "id": 454, + "origin_id": 394, + "origin_slot": 1, + "target_id": 393, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 493, + "origin_id": 369, + "origin_slot": 0, + "target_id": 395, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 458, + "origin_id": 394, + "origin_slot": 0, + "target_id": 395, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 461, + "origin_id": 393, + "origin_slot": 0, + "target_id": 395, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 460, + "origin_id": 395, + "origin_slot": 0, + "target_id": 396, + "target_slot": 0, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 464, + "origin_id": 397, + "origin_slot": 0, + "target_id": 396, + "target_slot": 1, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 476, + "origin_id": 398, + "origin_slot": 0, + "target_id": 396, + "target_slot": 4, + "type": "BOOLEAN" + }, + { + "id": 463, + "origin_id": 394, + "origin_slot": 0, + "target_id": 397, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 488, + "origin_id": 407, + "origin_slot": 0, + "target_id": 397, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 466, + "origin_id": 396, + "origin_slot": 0, + "target_id": 399, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 465, + "origin_id": 396, + "origin_slot": 1, + "target_id": 400, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 124, + "origin_id": 378, + "origin_slot": 1, + "target_id": 401, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 407, + "origin_id": 368, + "origin_slot": 0, + "target_id": 401, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 196, + "origin_id": 373, + "origin_slot": 0, + "target_id": 403, + "target_slot": 0, + "type": "CLIP_VISION" + }, + { + "id": 8, + "origin_id": 372, + "origin_slot": 0, + "target_id": 404, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 7, + "origin_id": 372, + "origin_slot": 0, + "target_id": 405, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 325, + "origin_id": 405, + "origin_slot": 0, + "target_id": 406, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 326, + "origin_id": 404, + "origin_slot": 0, + "target_id": 406, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 406, + "origin_id": 368, + "origin_slot": 0, + "target_id": 406, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 420, + "origin_id": 369, + "origin_slot": 0, + "target_id": 406, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 467, + "origin_id": 396, + "origin_slot": 0, + "target_id": 406, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 472, + "origin_id": 396, + "origin_slot": 1, + "target_id": 406, + "target_slot": 6, + "type": "IMAGE" + }, + { + "id": 404, + "origin_id": 403, + "origin_slot": 0, + "target_id": 406, + "target_slot": 7, + "type": "CLIP_VISION_OUTPUT" + }, + { + "id": 414, + "origin_id": 371, + "origin_slot": 0, + "target_id": 406, + "target_slot": 9, + "type": "INT" + }, + { + "id": 415, + "origin_id": 371, + "origin_slot": 1, + "target_id": 406, + "target_slot": 10, + "type": "INT" + }, + { + "id": 416, + "origin_id": 371, + "origin_slot": 2, + "target_id": 406, + "target_slot": 11, + "type": "INT" + }, + { + "id": 475, + "origin_id": 398, + "origin_slot": 0, + "target_id": 406, + "target_slot": 18, + "type": "BOOLEAN" + }, + { + "id": 489, + "origin_id": 394, + "origin_slot": 1, + "target_id": 407, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 409, + "origin_id": -10, + "origin_slot": 0, + "target_id": 370, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 469, + "origin_id": -10, + "origin_slot": 1, + "target_id": 397, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 408, + "origin_id": -10, + "origin_slot": 1, + "target_id": 403, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 473, + "origin_id": -10, + "origin_slot": 1, + "target_id": 406, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 499, + "origin_id": -10, + "origin_slot": 2, + "target_id": 405, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 502, + "origin_id": -10, + "origin_slot": 3, + "target_id": 391, + "target_slot": 0, + "type": "INT" + }, + { + "id": 504, + "origin_id": -10, + "origin_slot": 4, + "target_id": 398, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 505, + "origin_id": -10, + "origin_slot": 5, + "target_id": 387, + "target_slot": 0, + "type": "INT" + }, + { + "id": 506, + "origin_id": -10, + "origin_slot": 6, + "target_id": 388, + "target_slot": 0, + "type": "INT" + }, + { + "id": 507, + "origin_id": -10, + "origin_slot": 12, + "target_id": 402, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 508, + "origin_id": -10, + "origin_slot": 13, + "target_id": 374, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 509, + "origin_id": -10, + "origin_slot": 14, + "target_id": 367, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 510, + "origin_id": -10, + "origin_slot": 16, + "target_id": 372, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 511, + "origin_id": -10, + "origin_slot": 17, + "target_id": 368, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 512, + "origin_id": -10, + "origin_slot": 18, + "target_id": 373, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 513, + "origin_id": -10, + "origin_slot": 19, + "target_id": 393, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 514, + "origin_id": -10, + "origin_slot": 20, + "target_id": 407, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 515, + "origin_id": -10, + "origin_slot": 21, + "target_id": 394, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 522, + "origin_id": -10, + "origin_slot": 7, + "target_id": 386, + "target_slot": 2, + "type": "INT" + }, + { + "id": 541, + "origin_id": -10, + "origin_slot": 7, + "target_id": 392, + "target_slot": 1, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 542, + "origin_id": -10, + "origin_slot": 8, + "target_id": 406, + "target_slot": 17, + "type": "INT" + }, + { + "id": 543, + "origin_id": -10, + "origin_slot": 8, + "target_id": 392, + "target_slot": 2, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 544, + "origin_id": 401, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 547, + "origin_id": -10, + "origin_slot": 9, + "target_id": 406, + "target_slot": 13, + "type": "FLOAT" + }, + { + "id": 548, + "origin_id": -10, + "origin_slot": 10, + "target_id": 406, + "target_slot": 14, + "type": "FLOAT" + }, + { + "id": 549, + "origin_id": -10, + "origin_slot": 11, + "target_id": 406, + "target_slot": 15, + "type": "FLOAT" + }, + { + "id": 558, + "origin_id": -10, + "origin_slot": 22, + "target_id": 378, + "target_slot": 7, + "type": "INT" + }, + { + "id": 562, + "origin_id": 374, + "origin_slot": 0, + "target_id": 408, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 563, + "origin_id": 408, + "origin_slot": 0, + "target_id": 367, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 564, + "origin_id": 408, + "origin_slot": 0, + "target_id": 383, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 565, + "origin_id": -10, + "origin_slot": 15, + "target_id": 408, + "target_slot": 1, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Video generation and editing/Video Edit", + "description": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos." + } + ] + }, + "extra": { + "BlueprintDescription": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos.", + "BlueprintSearchAliases": [ + "character replacement", + "SCAIL-2", + "pose-free animation", + "video editing", + "cross-identity replacement" + ] + } +} \ No newline at end of file diff --git a/blueprints/Character Replacement (SCAIL-2 Extend).json b/blueprints/Character Replacement (SCAIL-2 Extend).json new file mode 100644 index 000000000..bf2f67dfd --- /dev/null +++ b/blueprints/Character Replacement (SCAIL-2 Extend).json @@ -0,0 +1,4461 @@ +{ + "revision": 0, + "last_node_id": 409, + "last_link_id": 0, + "nodes": [ + { + "id": 409, + "type": "2ebfb952-bd3b-43c3-9390-9ea73d41bd1f", + "pos": [ + 3420, + 5580 + ], + "size": [ + 530, + 1140 + ], + "flags": { + "collapsed": false + }, + "order": 4, + "mode": 0, + "inputs": [ + { + "label": "pose_video", + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "label": "reference_image", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": null + }, + { + "label": "previous_frames", + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "segment_index", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "replace_mode", + "name": "value_2", + "type": "BOOLEAN", + "widget": { + "name": "value_2" + }, + "link": null + }, + { + "label": "width", + "name": "value_3", + "type": "INT", + "widget": { + "name": "value_3" + }, + "link": null + }, + { + "label": "height", + "name": "value_4", + "type": "INT", + "widget": { + "name": "value_4" + }, + "link": null + }, + { + "label": "frame_count", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + }, + { + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": null + }, + { + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": null + }, + { + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": null + }, + { + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": null + }, + { + "label": "turbo_mode", + "name": "value_5", + "type": "BOOLEAN", + "widget": { + "name": "value_5" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "distill_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "label": "dpo_lora", + "name": "lora_name_1", + "type": "COMBO", + "widget": { + "name": "lora_name_1" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "clip_vision", + "name": "clip_name_1", + "type": "COMBO", + "widget": { + "name": "clip_name_1" + }, + "link": null + }, + { + "label": "sam3_video_object", + "name": "text_1", + "type": "STRING", + "widget": { + "name": "text_1" + }, + "link": null + }, + { + "label": "sam3_image_object", + "name": "text_2", + "type": "STRING", + "widget": { + "name": "text_2" + }, + "link": null + }, + { + "label": "sam3_model", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "363", + "text" + ], + [ + "346", + "value" + ], + [ + "353", + "value" + ], + [ + "342", + "value" + ], + [ + "343", + "value" + ], + [ + "341", + "length" + ], + [ + "364", + "previous_frame_count" + ], + [ + "364", + "pose_strength" + ], + [ + "364", + "pose_start" + ], + [ + "364", + "pose_end" + ], + [ + "360", + "value" + ], + [ + "329", + "unet_name" + ], + [ + "322", + "lora_name" + ], + [ + "366", + "lora_name" + ], + [ + "327", + "clip_name" + ], + [ + "323", + "vae_name" + ], + [ + "328", + "clip_name" + ], + [ + "348", + "text" + ], + [ + "365", + "text" + ], + [ + "349", + "ckpt_name" + ], + [ + "333", + "noise_seed" + ], + [ + "354", + "$$canvas-image-preview" + ], + [ + "355", + "$$canvas-image-preview" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Character Replacement (SCAIL-2 Extend)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "2ebfb952-bd3b-43c3-9390-9ea73d41bd1f", + "version": 1, + "state": { + "lastGroupId": 17, + "lastNodeId": 410, + "lastLinkId": 570, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Character Replacement (SCAIL-2 Extend)", + "inputNode": { + "id": -10, + "bounding": [ + -300, + 4240, + 173.015625, + 528 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 3570, + 4500, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "e9aedfaf-1991-4a75-ad9b-8c3a279b0488", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 409 + ], + "localized_name": "video", + "label": "pose_video", + "pos": [ + -150.984375, + 4264 + ] + }, + { + "id": "0badb26e-9abd-4e9c-b221-aab1237b8773", + "name": "images", + "type": "IMAGE", + "linkIds": [ + 469, + 408, + 473 + ], + "localized_name": "images", + "label": "reference_image", + "pos": [ + -150.984375, + 4284 + ] + }, + { + "id": "9f424e9e-d5a2-4dd9-9934-cc6f6fc8da09", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 486, + 485 + ], + "localized_name": "image", + "label": "previous_frames", + "pos": [ + -150.984375, + 4304 + ] + }, + { + "id": "098f15aa-a066-422e-b491-eaf140bafb2c", + "name": "text", + "type": "STRING", + "linkIds": [ + 499 + ], + "label": "prompt", + "pos": [ + -150.984375, + 4324 + ] + }, + { + "id": "861ea850-8329-491b-9413-2588cdd77bbd", + "name": "value", + "type": "INT", + "linkIds": [ + 502 + ], + "label": "segment_index", + "pos": [ + -150.984375, + 4344 + ] + }, + { + "id": "dcf3ebf8-1263-458a-966b-9e77a5ad704a", + "name": "value_2", + "type": "BOOLEAN", + "linkIds": [ + 504 + ], + "label": "replace_mode", + "pos": [ + -150.984375, + 4364 + ] + }, + { + "id": "dbee10ce-bb98-4733-88a0-d4cd5b13c691", + "name": "value_3", + "type": "INT", + "linkIds": [ + 505 + ], + "label": "width", + "pos": [ + -150.984375, + 4384 + ] + }, + { + "id": "363e3ac7-0096-4480-b875-1d6ff1b8a2d9", + "name": "value_4", + "type": "INT", + "linkIds": [ + 506 + ], + "label": "height", + "pos": [ + -150.984375, + 4404 + ] + }, + { + "id": "dc19eaf1-dd24-4c1e-9b08-6d8d98e0e7f2", + "name": "length", + "type": "INT", + "linkIds": [ + 550, + 557 + ], + "label": "frame_count", + "pos": [ + -150.984375, + 4424 + ] + }, + { + "id": "39d2f098-1740-419e-9c29-e4c4e4a7c3fd", + "name": "previous_frame_count", + "type": "INT", + "linkIds": [ + 551, + 556 + ], + "pos": [ + -150.984375, + 4444 + ] + }, + { + "id": "8d248225-a7f5-42c5-86bd-31eaf695f66e", + "name": "pose_strength", + "type": "FLOAT", + "linkIds": [ + 552 + ], + "pos": [ + -150.984375, + 4464 + ] + }, + { + "id": "04c56360-963a-48ba-944e-4864e2c0349c", + "name": "pose_start", + "type": "FLOAT", + "linkIds": [ + 553 + ], + "pos": [ + -150.984375, + 4484 + ] + }, + { + "id": "1ecd0315-d61f-4986-837c-27fb3f2d0470", + "name": "pose_end", + "type": "FLOAT", + "linkIds": [ + 554 + ], + "pos": [ + -150.984375, + 4504 + ] + }, + { + "id": "ddf07ba4-2837-40ef-925f-5996ea436334", + "name": "value_5", + "type": "BOOLEAN", + "linkIds": [ + 507 + ], + "label": "turbo_mode", + "pos": [ + -150.984375, + 4524 + ] + }, + { + "id": "ee8c796d-b326-40c6-9f9d-65f564053974", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 508 + ], + "pos": [ + -150.984375, + 4544 + ] + }, + { + "id": "6ccae991-51d7-4c8e-adc8-c7f6377d681b", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 509 + ], + "label": "distill_lora", + "pos": [ + -150.984375, + 4564 + ] + }, + { + "id": "1f1ac950-6ce5-4253-a266-edba58acd135", + "name": "lora_name_1", + "type": "COMBO", + "linkIds": [ + 569 + ], + "label": "dpo_lora", + "pos": [ + -150.984375, + 4584 + ] + }, + { + "id": "2e9c2347-93fe-462f-b5ef-51613fb52c85", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 510 + ], + "pos": [ + -150.984375, + 4604 + ] + }, + { + "id": "bc1822c0-7484-47b7-8d59-cf94788290f7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 511 + ], + "pos": [ + -150.984375, + 4624 + ] + }, + { + "id": "bbf2ee19-58b3-46f4-af6a-cd1557d60c8d", + "name": "clip_name_1", + "type": "COMBO", + "linkIds": [ + 512 + ], + "label": "clip_vision", + "pos": [ + -150.984375, + 4644 + ] + }, + { + "id": "556415bd-8ef3-4cfa-b182-0d7e3e4cd572", + "name": "text_1", + "type": "STRING", + "linkIds": [ + 513 + ], + "label": "sam3_video_object", + "pos": [ + -150.984375, + 4664 + ] + }, + { + "id": "c577930c-85a3-4e92-ac44-8ed639937217", + "name": "text_2", + "type": "STRING", + "linkIds": [ + 514 + ], + "label": "sam3_image_object", + "pos": [ + -150.984375, + 4684 + ] + }, + { + "id": "8f542c5e-da91-4dad-8ff8-4e81a42d5140", + "name": "ckpt_name", + "type": "COMBO", + "linkIds": [ + 515 + ], + "label": "sam3_model", + "pos": [ + -150.984375, + 4704 + ] + }, + { + "id": "9764a497-f066-429c-b77c-bcd5b8c7a68d", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 559 + ], + "pos": [ + -150.984375, + 4724 + ] + } + ], + "outputs": [ + { + "id": "7ae88834-7553-45ec-a4e8-bab7d5276b45", + "name": "output", + "type": "IMAGE", + "linkIds": [ + 546 + ], + "localized_name": "output", + "pos": [ + 3594, + 4524 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 322, + "type": "LoraLoaderModelOnly", + "pos": [ + 300, + 3590 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 566 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 509 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 398, + 425 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 0.8 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 323, + "type": "VAELoader", + "pos": [ + 300, + 4020 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 511 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 406, + 407 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 324, + "type": "ResizeImageMaskNode", + "pos": [ + 1280, + 4880 + ], + "size": [ + 270, + 160 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "input", + "name": "input", + "type": "IMAGE,MASK", + "link": 491 + }, + { + "localized_name": "resize_type", + "name": "resize_type", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "resize_type" + }, + "link": null + }, + { + "localized_name": "width", + "name": "resize_type.width", + "type": "INT", + "widget": { + "name": "resize_type.width" + }, + "link": 444 + }, + { + "localized_name": "height", + "name": "resize_type.height", + "type": "INT", + "widget": { + "name": "resize_type.height" + }, + "link": 446 + }, + { + "localized_name": "crop", + "name": "resize_type.crop", + "type": "COMBO", + "widget": { + "name": "resize_type.crop" + }, + "link": null + }, + { + "localized_name": "scale_method", + "name": "scale_method", + "type": "COMBO", + "widget": { + "name": "scale_method" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "resized", + "name": "resized", + "type": "*", + "links": [ + 420, + 492, + 493 + ] + } + ], + "properties": { + "Node name for S&R": "ResizeImageMaskNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "scale dimensions", + 512, + 512, + "center", + "area" + ] + }, + { + "id": 325, + "type": "GetVideoComponents", + "pos": [ + 270, + 4520 + ], + "size": [ + 230, + 90 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 409 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 490 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 326, + "type": "GetImageSize", + "pos": [ + 1640, + 4520 + ], + "size": [ + 240, + 190 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 492 + } + ], + "outputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "links": [ + 414 + ] + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "links": [ + 415 + ] + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "links": [ + 416 + ] + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 327, + "type": "CLIPLoader", + "pos": [ + 300, + 3790 + ], + "size": [ + 590, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 510 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 7, + 8 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "directory": "text_encoders" + } + ] + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 328, + "type": "CLIPVisionLoader", + "pos": [ + 300, + 4230 + ], + "size": [ + 590, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 512 + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 196 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "clip_vision_h.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/clip_vision/clip_vision_h.safetensors", + "directory": "clip_vision" + } + ] + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ] + }, + { + "id": 329, + "type": "UNETLoader", + "pos": [ + 300, + 3190 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 508 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 568 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_14B_SCAIL_2_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/diffusion_models/wan2.1_14B_SCAIL_2_fp16.safetensors", + "directory": "diffusion_models" + } + ] + }, + "widgets_values": [ + "wan2.1_14B_SCAIL_2_fp16.safetensors", + "default" + ] + }, + { + "id": 330, + "type": "ModelSamplingSD3", + "pos": [ + 2820, + 3430 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 424 + }, + { + "localized_name": "shift", + "name": "shift", + "type": "FLOAT", + "widget": { + "name": "shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 417 + ] + } + ], + "properties": { + "Node name for S&R": "ModelSamplingSD3", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 331, + "type": "KSamplerSelect", + "pos": [ + 2830, + 3580 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 419 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 332, + "type": "BasicScheduler", + "pos": [ + 2830, + 3730 + ], + "size": [ + 260, + 170 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 398 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 428 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 418 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 333, + "type": "SamplerCustom", + "pos": [ + 3140, + 3430 + ], + "size": [ + 270, + 670 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 417 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 333 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 334 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 419 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 418 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 335 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 559 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 431 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [ + 124 + ] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true, + 1, + "fixed", + 1 + ] + }, + { + "id": 334, + "type": "PrimitiveInt", + "pos": [ + 2090, + 3760 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 427 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 335, + "type": "PrimitiveFloat", + "pos": [ + 2090, + 3930 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 430 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 336, + "type": "PrimitiveInt", + "pos": [ + 2090, + 3390 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 426 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 337, + "type": "PrimitiveFloat", + "pos": [ + 2090, + 3540 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 429 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 338, + "type": "ComfySwitchNode", + "pos": [ + 2430, + 3370 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 567 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 425 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 432 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 424 + ] + } + ], + "title": "Switch (Model)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 339, + "type": "ComfySwitchNode", + "pos": [ + 2430, + 3560 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 426 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 427 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 433 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 428 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 340, + "type": "ComfySwitchNode", + "pos": [ + 2430, + 3740 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 429 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 430 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 434 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 431 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 341, + "type": "ImageFromBatch", + "pos": [ + 880, + 4510 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 490 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": 450 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 550 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 491 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0, + 81 + ] + }, + { + "id": 342, + "type": "PrimitiveInt", + "pos": [ + 250, + 4840 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 505 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Int (Width)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 896, + "fixed" + ] + }, + { + "id": 343, + "type": "PrimitiveInt", + "pos": [ + 250, + 5020 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 506 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 445 + ] + } + ], + "title": "Int (Height)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + "fixed" + ] + }, + { + "id": 344, + "type": "ComfyMathExpression", + "pos": [ + 690, + 4870 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 441 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 444 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 345, + "type": "ComfyMathExpression", + "pos": [ + 690, + 5050 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 445 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 446 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 346, + "type": "PrimitiveInt", + "pos": [ + 270, + 4580 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 502 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 447 + ] + } + ], + "title": "Int (segment index)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 2, + "fixed" + ] + }, + { + "id": 347, + "type": "ComfyMathExpression", + "pos": [ + 580, + 4610 + ], + "size": [ + 230, + 120 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 447 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 557 + }, + { + "label": "c", + "localized_name": "values.c", + "name": "values.c", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 556 + }, + { + "label": "d", + "localized_name": "values.d", + "name": "values.d", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 450 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(b - c) * (a - 1)" + ] + }, + { + "id": 348, + "type": "CLIPTextEncode", + "pos": [ + 660, + 5380 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 454 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 513 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 461 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 349, + "type": "CheckpointLoaderSimple", + "pos": [ + 270, + 5360 + ], + "size": [ + 330, + 160 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "ckpt_name", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": 515 + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 458, + 463 + ] + }, + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 454, + 489 + ] + }, + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [] + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "models": [ + { + "name": "sam3.1_multiplex_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/sam3.1/resolve/main/checkpoints/sam3.1_multiplex_fp16.safetensors", + "directory": "checkpoints" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "sam3.1_multiplex_fp16.safetensors" + ] + }, + { + "id": 350, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5340 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 493 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 458 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 461 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 460 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 351, + "type": "SCAIL2ColoredMask", + "pos": [ + 1550, + 5490 + ], + "size": [ + 370, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "driving_track_data", + "name": "driving_track_data", + "type": "SAM3_TRACK_DATA", + "link": 460 + }, + { + "localized_name": "ref_track_data", + "name": "ref_track_data", + "shape": 7, + "type": "SAM3_TRACK_DATA", + "link": 464 + }, + { + "localized_name": "object_indices", + "name": "object_indices", + "type": "STRING", + "widget": { + "name": "object_indices" + }, + "link": null + }, + { + "localized_name": "sort_by", + "name": "sort_by", + "type": "COMBO", + "widget": { + "name": "sort_by" + }, + "link": null + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 476 + } + ], + "outputs": [ + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "type": "IMAGE", + "links": [ + 466, + 467 + ] + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "type": "IMAGE", + "links": [ + 465, + 472 + ] + } + ], + "properties": { + "Node name for S&R": "SCAIL2ColoredMask", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "left_to_right", + true + ] + }, + { + "id": 352, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5640 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 469 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 463 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 488 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 464 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 353, + "type": "PrimitiveBoolean", + "pos": [ + 1660, + 4030 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 504 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 475, + 476 + ] + } + ], + "title": "Boolean （Replace Mode）", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 354, + "type": "PreviewImage", + "pos": [ + 2060, + 4500 + ], + "size": [ + 350, + 1190 + ], + "flags": {}, + "order": 32, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 466 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 355, + "type": "PreviewImage", + "pos": [ + 2460, + 4500 + ], + "size": [ + 230, + 310 + ], + "flags": {}, + "order": 33, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 465 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 356, + "type": "VAEDecode", + "pos": [ + 2920, + 4510 + ], + "size": [ + 270, + 100 + ], + "flags": { + "collapsed": false + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 124 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 407 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 483 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 357, + "type": "ImageFromBatch", + "pos": [ + 2920, + 4680 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 483 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": null + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 484 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5, + 4096 + ] + }, + { + "id": 358, + "type": "ColorTransfer", + "pos": [ + 2920, + 5050 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "image_target", + "name": "image_target", + "type": "IMAGE", + "link": 484 + }, + { + "localized_name": "image_ref", + "name": "image_ref", + "type": "IMAGE", + "link": 482 + }, + { + "localized_name": "method", + "name": "method", + "type": "COMBO", + "widget": { + "name": "method" + }, + "link": null + }, + { + "localized_name": "source_stats", + "name": "source_stats", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "source_stats" + }, + "link": null + }, + { + "localized_name": "strength", + "name": "strength", + "type": "FLOAT", + "widget": { + "name": "strength" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "links": [ + 546 + ] + } + ], + "properties": { + "Node name for S&R": "ColorTransfer", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "reinhard_lab", + "per_frame", + 1 + ] + }, + { + "id": 359, + "type": "ImageFromBatch", + "pos": [ + 2920, + 4870 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 486 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": null + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 482 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + -1, + 1 + ] + }, + { + "id": 360, + "type": "PrimitiveBoolean", + "pos": [ + 2440, + 3950 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 507 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 432, + 433, + 434 + ] + } + ], + "title": "Boolean (Enable Distill LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 361, + "type": "CLIPVisionEncode", + "pos": [ + 1230, + 4310 + ], + "size": [ + 230, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 196 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 408 + }, + { + "localized_name": "crop", + "name": "crop", + "type": "COMBO", + "widget": { + "name": "crop" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION_OUTPUT", + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "links": [ + 404 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 362, + "type": "CLIPTextEncode", + "pos": [ + 1030, + 4020 + ], + "size": [ + 520, + 210 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 326 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 363, + "type": "CLIPTextEncode", + "pos": [ + 1020, + 3180 + ], + "size": [ + 520, + 720 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 499 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 325 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "A young woman with dark hair tied in a neat high bun, with a few loose strands framing her face, is dancing outdoors on a sunny coastal hillside. She has a normal-sized head and a slim face, with no hat, no headwear, and no oversized hair volume. She wears a fitted black long-sleeve crop top with a shoulder cutout, extremely baggy black cargo pants with straps and pockets, and chunky black combat boots. She performs energetic dance moves with one leg lifted and arms extended, moving naturally in front of a large tree, a small white stone house with a terracotta roof, and a bright blue sea under a clear sky with light clouds." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 364, + "type": "WanSCAILToVideo", + "pos": [ + 1650, + 3310 + ], + "size": [ + 310, + 580 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 325 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 326 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 406 + }, + { + "localized_name": "pose_video", + "name": "pose_video", + "shape": 7, + "type": "IMAGE", + "link": 420 + }, + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "shape": 7, + "type": "IMAGE", + "link": 467 + }, + { + "localized_name": "reference_image", + "name": "reference_image", + "shape": 7, + "type": "IMAGE", + "link": 473 + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "shape": 7, + "type": "IMAGE", + "link": 472 + }, + { + "localized_name": "clip_vision_output", + "name": "clip_vision_output", + "shape": 7, + "type": "CLIP_VISION_OUTPUT", + "link": 404 + }, + { + "localized_name": "previous_frames", + "name": "previous_frames", + "shape": 7, + "type": "IMAGE", + "link": 485 + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 414 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 415 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 416 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "pose_strength", + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": 552 + }, + { + "localized_name": "pose_start", + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": 553 + }, + { + "localized_name": "pose_end", + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": 554 + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "widget": { + "name": "video_frame_offset" + }, + "link": null + }, + { + "localized_name": "previous_frame_count", + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": 551 + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 475 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 333 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 334 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 335 + ] + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanSCAILToVideo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + 896, + 65, + 1, + 1, + 0, + 1, + 0, + 5, + true + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 365, + "type": "CLIPTextEncode", + "pos": [ + 670, + 5710 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 489 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 514 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 488 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 366, + "type": "LoraLoaderModelOnly", + "pos": [ + 300, + 3390 + ], + "size": [ + 580, + 140 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 568 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 569 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 566, + 567 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/loras/wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + 1 + ] + } + ], + "groups": [ + { + "id": 3, + "title": "Models", + "bounding": [ + 240, + 3100, + 720, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 990, + 3100, + 580, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Sampling", + "bounding": [ + 2770, + 3100, + 700, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "SCAIL Conditioning", + "bounding": [ + 1590, + 3100, + 430, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 9, + "title": "Switch Settings", + "bounding": [ + 2050, + 3100, + 690, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Original Settings", + "bounding": [ + 2080, + 3330, + 300, + 340 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Distill LoRA Settings", + "bounding": [ + 2080, + 3690, + 300, + 370 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 10, + "title": "Create Mask", + "bounding": [ + 240, + 5250, + 1780, + 670 + ], + "flags": {} + }, + { + "id": 11, + "title": "Resize Images", + "bounding": [ + 240, + 4770, + 1780, + 450 + ], + "flags": {} + }, + { + "id": 12, + "title": "Trim Frames from Video", + "bounding": [ + 240, + 4430, + 1780, + 310 + ], + "flags": {} + }, + { + "id": 13, + "title": "Preview Masks", + "bounding": [ + 2050, + 4430, + 690, + 1490 + ], + "flags": {} + }, + { + "id": 14, + "title": "Group", + "bounding": [ + 2770, + 4430, + 700, + 1490 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 491, + "origin_id": 341, + "origin_slot": 0, + "target_id": 324, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 444, + "origin_id": 344, + "origin_slot": 1, + "target_id": 324, + "target_slot": 2, + "type": "INT" + }, + { + "id": 446, + "origin_id": 345, + "origin_slot": 1, + "target_id": 324, + "target_slot": 3, + "type": "INT" + }, + { + "id": 492, + "origin_id": 324, + "origin_slot": 0, + "target_id": 326, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 424, + "origin_id": 338, + "origin_slot": 0, + "target_id": 330, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 398, + "origin_id": 322, + "origin_slot": 0, + "target_id": 332, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 428, + "origin_id": 339, + "origin_slot": 0, + "target_id": 332, + "target_slot": 2, + "type": "INT" + }, + { + "id": 417, + "origin_id": 330, + "origin_slot": 0, + "target_id": 333, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 333, + "origin_id": 364, + "origin_slot": 0, + "target_id": 333, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 334, + "origin_id": 364, + "origin_slot": 1, + "target_id": 333, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 419, + "origin_id": 331, + "origin_slot": 0, + "target_id": 333, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 418, + "origin_id": 332, + "origin_slot": 0, + "target_id": 333, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 335, + "origin_id": 364, + "origin_slot": 2, + "target_id": 333, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 431, + "origin_id": 340, + "origin_slot": 0, + "target_id": 333, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 425, + "origin_id": 322, + "origin_slot": 0, + "target_id": 338, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 432, + "origin_id": 360, + "origin_slot": 0, + "target_id": 338, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 426, + "origin_id": 336, + "origin_slot": 0, + "target_id": 339, + "target_slot": 0, + "type": "INT" + }, + { + "id": 427, + "origin_id": 334, + "origin_slot": 0, + "target_id": 339, + "target_slot": 1, + "type": "INT" + }, + { + "id": 433, + "origin_id": 360, + "origin_slot": 0, + "target_id": 339, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 429, + "origin_id": 337, + "origin_slot": 0, + "target_id": 340, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 430, + "origin_id": 335, + "origin_slot": 0, + "target_id": 340, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 434, + "origin_id": 360, + "origin_slot": 0, + "target_id": 340, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 490, + "origin_id": 325, + "origin_slot": 0, + "target_id": 341, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 450, + "origin_id": 347, + "origin_slot": 1, + "target_id": 341, + "target_slot": 1, + "type": "INT" + }, + { + "id": 441, + "origin_id": 342, + "origin_slot": 0, + "target_id": 344, + "target_slot": 0, + "type": "INT" + }, + { + "id": 445, + "origin_id": 343, + "origin_slot": 0, + "target_id": 345, + "target_slot": 0, + "type": "INT" + }, + { + "id": 447, + "origin_id": 346, + "origin_slot": 0, + "target_id": 347, + "target_slot": 0, + "type": "INT" + }, + { + "id": 454, + "origin_id": 349, + "origin_slot": 1, + "target_id": 348, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 493, + "origin_id": 324, + "origin_slot": 0, + "target_id": 350, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 458, + "origin_id": 349, + "origin_slot": 0, + "target_id": 350, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 461, + "origin_id": 348, + "origin_slot": 0, + "target_id": 350, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 460, + "origin_id": 350, + "origin_slot": 0, + "target_id": 351, + "target_slot": 0, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 464, + "origin_id": 352, + "origin_slot": 0, + "target_id": 351, + "target_slot": 1, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 476, + "origin_id": 353, + "origin_slot": 0, + "target_id": 351, + "target_slot": 4, + "type": "BOOLEAN" + }, + { + "id": 463, + "origin_id": 349, + "origin_slot": 0, + "target_id": 352, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 488, + "origin_id": 365, + "origin_slot": 0, + "target_id": 352, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 466, + "origin_id": 351, + "origin_slot": 0, + "target_id": 354, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 465, + "origin_id": 351, + "origin_slot": 1, + "target_id": 355, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 124, + "origin_id": 333, + "origin_slot": 1, + "target_id": 356, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 407, + "origin_id": 323, + "origin_slot": 0, + "target_id": 356, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 483, + "origin_id": 356, + "origin_slot": 0, + "target_id": 357, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 484, + "origin_id": 357, + "origin_slot": 0, + "target_id": 358, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 482, + "origin_id": 359, + "origin_slot": 0, + "target_id": 358, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 196, + "origin_id": 328, + "origin_slot": 0, + "target_id": 361, + "target_slot": 0, + "type": "CLIP_VISION" + }, + { + "id": 8, + "origin_id": 327, + "origin_slot": 0, + "target_id": 362, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 7, + "origin_id": 327, + "origin_slot": 0, + "target_id": 363, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 325, + "origin_id": 363, + "origin_slot": 0, + "target_id": 364, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 326, + "origin_id": 362, + "origin_slot": 0, + "target_id": 364, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 406, + "origin_id": 323, + "origin_slot": 0, + "target_id": 364, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 420, + "origin_id": 324, + "origin_slot": 0, + "target_id": 364, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 467, + "origin_id": 351, + "origin_slot": 0, + "target_id": 364, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 472, + "origin_id": 351, + "origin_slot": 1, + "target_id": 364, + "target_slot": 6, + "type": "IMAGE" + }, + { + "id": 404, + "origin_id": 361, + "origin_slot": 0, + "target_id": 364, + "target_slot": 7, + "type": "CLIP_VISION_OUTPUT" + }, + { + "id": 414, + "origin_id": 326, + "origin_slot": 0, + "target_id": 364, + "target_slot": 9, + "type": "INT" + }, + { + "id": 415, + "origin_id": 326, + "origin_slot": 1, + "target_id": 364, + "target_slot": 10, + "type": "INT" + }, + { + "id": 416, + "origin_id": 326, + "origin_slot": 2, + "target_id": 364, + "target_slot": 11, + "type": "INT" + }, + { + "id": 475, + "origin_id": 353, + "origin_slot": 0, + "target_id": 364, + "target_slot": 18, + "type": "BOOLEAN" + }, + { + "id": 489, + "origin_id": 349, + "origin_slot": 1, + "target_id": 365, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 409, + "origin_id": -10, + "origin_slot": 0, + "target_id": 325, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 469, + "origin_id": -10, + "origin_slot": 1, + "target_id": 352, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 408, + "origin_id": -10, + "origin_slot": 1, + "target_id": 361, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 473, + "origin_id": -10, + "origin_slot": 1, + "target_id": 364, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 486, + "origin_id": -10, + "origin_slot": 2, + "target_id": 359, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 485, + "origin_id": -10, + "origin_slot": 2, + "target_id": 364, + "target_slot": 8, + "type": "IMAGE" + }, + { + "id": 499, + "origin_id": -10, + "origin_slot": 3, + "target_id": 363, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 502, + "origin_id": -10, + "origin_slot": 4, + "target_id": 346, + "target_slot": 0, + "type": "INT" + }, + { + "id": 504, + "origin_id": -10, + "origin_slot": 5, + "target_id": 353, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 505, + "origin_id": -10, + "origin_slot": 6, + "target_id": 342, + "target_slot": 0, + "type": "INT" + }, + { + "id": 506, + "origin_id": -10, + "origin_slot": 7, + "target_id": 343, + "target_slot": 0, + "type": "INT" + }, + { + "id": 507, + "origin_id": -10, + "origin_slot": 13, + "target_id": 360, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 508, + "origin_id": -10, + "origin_slot": 14, + "target_id": 329, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 509, + "origin_id": -10, + "origin_slot": 15, + "target_id": 322, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 510, + "origin_id": -10, + "origin_slot": 17, + "target_id": 327, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 511, + "origin_id": -10, + "origin_slot": 18, + "target_id": 323, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 512, + "origin_id": -10, + "origin_slot": 19, + "target_id": 328, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 513, + "origin_id": -10, + "origin_slot": 20, + "target_id": 348, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 514, + "origin_id": -10, + "origin_slot": 21, + "target_id": 365, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 515, + "origin_id": -10, + "origin_slot": 22, + "target_id": 349, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 546, + "origin_id": 358, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 550, + "origin_id": -10, + "origin_slot": 8, + "target_id": 341, + "target_slot": 2, + "type": "INT" + }, + { + "id": 551, + "origin_id": -10, + "origin_slot": 9, + "target_id": 364, + "target_slot": 17, + "type": "INT" + }, + { + "id": 552, + "origin_id": -10, + "origin_slot": 10, + "target_id": 364, + "target_slot": 13, + "type": "FLOAT" + }, + { + "id": 553, + "origin_id": -10, + "origin_slot": 11, + "target_id": 364, + "target_slot": 14, + "type": "FLOAT" + }, + { + "id": 554, + "origin_id": -10, + "origin_slot": 12, + "target_id": 364, + "target_slot": 15, + "type": "FLOAT" + }, + { + "id": 556, + "origin_id": -10, + "origin_slot": 9, + "target_id": 347, + "target_slot": 2, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 557, + "origin_id": -10, + "origin_slot": 8, + "target_id": 347, + "target_slot": 1, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 559, + "origin_id": -10, + "origin_slot": 23, + "target_id": 333, + "target_slot": 7, + "type": "INT" + }, + { + "id": 566, + "origin_id": 366, + "origin_slot": 0, + "target_id": 322, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 567, + "origin_id": 366, + "origin_slot": 0, + "target_id": 338, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 568, + "origin_id": 329, + "origin_slot": 0, + "target_id": 366, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 569, + "origin_id": -10, + "origin_slot": 16, + "target_id": 366, + "target_slot": 1, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Video generation and editing/Video Edit", + "description": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos." + } + ] + }, + "extra": { + "BlueprintDescription": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos.", + "BlueprintSearchAliases": [ + "character replacement", + "SCAIL-2 extend", + "video character swap" + ] + } +} \ No newline at end of file diff --git a/blueprints/Image Depth Estimation (Depth Anything 3).json b/blueprints/Image Depth Estimation (Depth Anything 3).json new file mode 100644 index 000000000..3c5f24845 --- /dev/null +++ b/blueprints/Image Depth Estimation (Depth Anything 3).json @@ -0,0 +1,569 @@ +{ + "revision": 0, + "last_node_id": 89, + "last_link_id": 0, + "nodes": [ + { + "id": 89, + "type": "85e595bd-af9e-40ee-85c5-b98bb15da47a", + "pos": [ + 320, + 520 + ], + "size": [ + 400, + 360 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": null + }, + { + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": null + }, + { + "label": "output_type", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": null + }, + { + "label": "output_normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": null + }, + { + "label": "apply_sky_clip", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": null + }, + { + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "87", + "resolution" + ], + [ + "87", + "resize_method" + ], + [ + "86", + "output" + ], + [ + "86", + "output.normalization" + ], + [ + "86", + "output.apply_sky_clip" + ], + [ + "88", + "model_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Image Depth Estimation (Depth Anything 3)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "85e595bd-af9e-40ee-85c5-b98bb15da47a", + "version": 1, + "state": { + "lastGroupId": 4, + "lastNodeId": 89, + "lastLinkId": 109, + "lastRerouteId": 0 + }, + "revision": 2, + "config": {}, + "name": "Image Depth Estimation (Depth Anything 3)", + "inputNode": { + "id": -10, + "bounding": [ + 400, + 90, + 166.998046875, + 188 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1250, + 146, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "43cf3118-495a-487d-8eb3-a17c7e92f64f", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 19 + ], + "localized_name": "image", + "pos": [ + 542.998046875, + 114 + ] + }, + { + "id": "1089a0a1-6db1-45a8-84b0-0bfdc2ed920a", + "name": "resolution", + "type": "INT", + "linkIds": [ + 22 + ], + "pos": [ + 542.998046875, + 134 + ] + }, + { + "id": "25fb64ac-26d5-466d-995b-6d51b9afa2c4", + "name": "resize_method", + "type": "COMBO", + "linkIds": [ + 23 + ], + "pos": [ + 542.998046875, + 154 + ] + }, + { + "id": "8acafb7c-6c8b-46b3-9d74-c563498a3af1", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "linkIds": [ + 24 + ], + "label": "output_type", + "pos": [ + 542.998046875, + 174 + ] + }, + { + "id": "1da5009b-4648-43e8-a257-16426630cf22", + "name": "output.normalization", + "type": "COMBO", + "linkIds": [ + 25 + ], + "label": "output_normalization", + "pos": [ + 542.998046875, + 194 + ] + }, + { + "id": "fd7edb33-5fb1-4538-a411-26e5039a9321", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "linkIds": [ + 26 + ], + "label": "apply_sky_clip", + "pos": [ + 542.998046875, + 214 + ] + }, + { + "id": "b5be4c8a-b833-4f1e-8c94-3ed1dd722190", + "name": "model_name", + "type": "COMBO", + "linkIds": [ + 106 + ], + "pos": [ + 542.998046875, + 234 + ] + } + ], + "outputs": [ + { + "id": "478ab537-63bc-4d74-a9f0-c975f550880f", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 7 + ], + "localized_name": "IMAGE", + "pos": [ + 1274, + 170 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 86, + "type": "DA3Render", + "pos": [ + 800, + 310 + ], + "size": [ + 380, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "link": 12 + }, + { + "localized_name": "output", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": 24 + }, + { + "localized_name": "output.normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": 25 + }, + { + "localized_name": "output.apply_sky_clip", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": 26 + }, + { + "name": "geometry", + "type": "DA3_GEOMETRY", + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Render", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + "depth", + "v2_style", + false + ] + }, + { + "id": 87, + "type": "DA3Inference", + "pos": [ + 800, + 50 + ], + "size": [ + 390, + 130 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_model", + "name": "da3_model", + "type": "DA3_MODEL", + "link": 107 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 19 + }, + { + "localized_name": "resolution", + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": 22 + }, + { + "localized_name": "resize_method", + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": 23 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "slot_index": 0, + "links": [ + 12 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Inference", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + 504, + "upper_bound_resize", + "mono" + ] + }, + { + "id": 88, + "type": "LoadDA3Model", + "pos": [ + 810, + -160 + ], + "size": [ + 400, + 140 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "model_name", + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": 106 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "DA3_MODEL", + "name": "DA3_MODEL", + "type": "DA3_MODEL", + "links": [ + 107 + ] + } + ], + "properties": { + "Node name for S&R": "LoadDA3Model", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "depth_anything_3_mono_large.safetensors", + "url": "https://huggingface.co/Comfy-Org/Depth-Anything-3/resolve/main/geometry_estimation/depth_anything_3_mono_large.safetensors", + "directory": "geometry_estimation" + } + ] + }, + "widgets_values": [ + "depth_anything_3_mono_large.safetensors", + "default" + ] + } + ], + "groups": [], + "links": [ + { + "id": 12, + "origin_id": 87, + "origin_slot": 0, + "target_id": 86, + "target_slot": 0, + "type": "DA3_GEOMETRY" + }, + { + "id": 19, + "origin_id": -10, + "origin_slot": 0, + "target_id": 87, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 7, + "origin_id": 86, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 22, + "origin_id": -10, + "origin_slot": 1, + "target_id": 87, + "target_slot": 2, + "type": "INT" + }, + { + "id": 23, + "origin_id": -10, + "origin_slot": 2, + "target_id": 87, + "target_slot": 3, + "type": "COMBO" + }, + { + "id": 24, + "origin_id": -10, + "origin_slot": 3, + "target_id": 86, + "target_slot": 1, + "type": "COMFY_DYNAMICCOMBO_V3" + }, + { + "id": 25, + "origin_id": -10, + "origin_slot": 4, + "target_id": 86, + "target_slot": 2, + "type": "COMBO" + }, + { + "id": 26, + "origin_id": -10, + "origin_slot": 5, + "target_id": 86, + "target_slot": 3, + "type": "BOOLEAN" + }, + { + "id": 106, + "origin_id": -10, + "origin_slot": 6, + "target_id": 88, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 107, + "origin_id": 88, + "origin_slot": 0, + "target_id": 87, + "target_slot": 0, + "type": "DA3_MODEL" + } + ], + "extra": {}, + "category": "Conditioning & Preprocessors/Depth", + "description": "This subgraph takes an input image and produces a depth map using the Depth Anything 3 model, which recovers spatially consistent geometry from any number of views. It is ideal for single or multi-view images, videos, and 3D scenes where accurate depth estimation is needed for tasks like SLAM, novel view synthesis, or spatial perception. The model uses a plain transformer backbone and supports both monocular and multi-view inputs without." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph takes an input image and produces a depth map using the Depth Anything 3 model, which recovers spatially consistent geometry from any number of views. It is ideal for single or multi-view images, videos, and 3D scenes where accurate depth estimation is needed for tasks like SLAM, novel view synthesis, or spatial perception. The model uses a plain transformer backbone and supports both monocular and multi-view inputs without." + } +} \ No newline at end of file diff --git a/blueprints/Image Edit (Bernini-R).json b/blueprints/Image Edit (Bernini-R).json new file mode 100644 index 000000000..9ad7867a4 --- /dev/null +++ b/blueprints/Image Edit (Bernini-R).json @@ -0,0 +1,3549 @@ +{ + "revision": 0, + "last_node_id": 76, + "last_link_id": 0, + "nodes": [ + { + "id": 76, + "type": "64670aa5-a932-4e9f-a299-3bfef6ebc043", + "pos": [ + 3240, + 4340 + ], + "size": [ + 510, + 850 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "source_image", + "name": "source_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "label": "reference_image", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "task_type", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": null + }, + { + "label": "enable_turbo_mode", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "high_noise_model", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "low_noise_model", + "name": "unet_name_1", + "type": "COMBO", + "widget": { + "name": "unet_name_1" + }, + "link": null + }, + { + "label": "lightning_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "120", + "value" + ], + [ + "54", + "choice" + ], + [ + "50", + "width" + ], + [ + "50", + "height" + ], + [ + "50", + "ref_max_size" + ], + [ + "70", + "value" + ], + [ + "5", + "unet_name" + ], + [ + "12", + "unet_name" + ], + [ + "11", + "lora_name" + ], + [ + "9", + "clip_name" + ], + [ + "7", + "vae_name" + ], + [ + "19", + "noise_seed" + ], + [ + "50", + "length" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Image Edit (Bernini-R)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "64670aa5-a932-4e9f-a299-3bfef6ebc043", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 157, + "lastLinkId": 308, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Image Edit (Bernini-R)", + "inputNode": { + "id": -10, + "bounding": [ + -60, + 4230, + 159.744140625, + 368 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 4720, + 4220, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "3df44016-bb49-40a2-a1dc-47d750440516", + "name": "source_video", + "type": "IMAGE", + "linkIds": [ + 294 + ], + "label": "source_image", + "pos": [ + 75.744140625, + 4254 + ] + }, + { + "id": "9e960570-35a6-4f48-bfa3-5f6ded6ede10", + "name": "reference_video", + "type": "IMAGE", + "linkIds": [ + 162 + ], + "pos": [ + 75.744140625, + 4274 + ] + }, + { + "id": "4ddf8f1f-b92a-4802-96cd-1379ce4dcba3", + "name": "reference_images.reference_image_0", + "type": "IMAGE", + "linkIds": [ + 163 + ], + "label": "reference_image", + "pos": [ + 75.744140625, + 4294 + ] + }, + { + "id": "fb6d520b-6281-4b05-8a36-64ed7ef58850", + "name": "text", + "type": "STRING", + "linkIds": [ + 308 + ], + "label": "prompt", + "pos": [ + 75.744140625, + 4314 + ] + }, + { + "id": "8005758c-ac19-4baf-8895-4661b2bf9327", + "name": "choice", + "type": "COMBO", + "linkIds": [ + 165 + ], + "label": "task_type", + "pos": [ + 75.744140625, + 4334 + ] + }, + { + "id": "8f7316d5-b98d-4702-83db-6f57fd8804c6", + "name": "width", + "type": "INT", + "linkIds": [ + 166 + ], + "pos": [ + 75.744140625, + 4354 + ] + }, + { + "id": "2dc22817-de2d-4d0f-8763-71795668c1d3", + "name": "height", + "type": "INT", + "linkIds": [ + 167 + ], + "pos": [ + 75.744140625, + 4374 + ] + }, + { + "id": "ff81d4ba-46f0-40e3-995d-50df973d951c", + "name": "ref_max_size", + "type": "INT", + "linkIds": [ + 169 + ], + "pos": [ + 75.744140625, + 4394 + ] + }, + { + "id": "2c12a5f5-e9d9-45e7-bb62-bbf7a352731f", + "name": "value", + "type": "BOOLEAN", + "linkIds": [ + 170 + ], + "label": "enable_turbo_mode", + "pos": [ + 75.744140625, + 4414 + ] + }, + { + "id": "ff09c771-83a5-4d92-bc16-006e6eb19406", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 171 + ], + "label": "high_noise_model", + "pos": [ + 75.744140625, + 4434 + ] + }, + { + "id": "67aed2f7-f407-4f55-add0-6ae669a069dc", + "name": "unet_name_1", + "type": "COMBO", + "linkIds": [ + 173 + ], + "label": "low_noise_model", + "pos": [ + 75.744140625, + 4454 + ] + }, + { + "id": "7f456845-cdba-46f0-829c-31b72d4a9038", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 174, + 175 + ], + "label": "lightning_lora", + "pos": [ + 75.744140625, + 4474 + ] + }, + { + "id": "b7332485-fd38-4522-8192-ea862afe9a35", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 176 + ], + "pos": [ + 75.744140625, + 4494 + ] + }, + { + "id": "3328f3bc-c4ce-467e-830e-0e394c8da2b7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 177 + ], + "pos": [ + 75.744140625, + 4514 + ] + }, + { + "id": "c8a71e1e-2d0b-4f9a-afd8-4186ba4cc241", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 283 + ], + "pos": [ + 75.744140625, + 4534 + ] + }, + { + "id": "3663f935-0959-4cb3-923d-d44736198056", + "name": "length", + "type": "INT", + "linkIds": [ + 295 + ], + "pos": [ + 75.744140625, + 4554 + ] + } + ], + "outputs": [ + { + "id": "5621984a-f548-41d5-b812-481a9122dd81", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 297 + ], + "pos": [ + 4744, + 4244 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 9, + "type": "CLIPLoader", + "pos": [ + 890, + 5030 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 176 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 7, + 8 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors?download=true", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 7, + "type": "VAELoader", + "pos": [ + 890, + 5260 + ], + "size": [ + 670, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 177 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 99, + 119 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors?download=true", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 29, + "type": "LoraLoaderModelOnly", + "pos": [ + 890, + 4810 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 128 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 175 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 132 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 1.5 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 4, + "type": "CLIPTextEncode", + "pos": [ + 1670, + 4670 + ], + "size": [ + 700, + 240 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 118 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 17, + "type": "SplitSigmas", + "pos": [ + 3240, + 4450 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 33 + }, + { + "localized_name": "step", + "name": "step", + "type": "INT", + "widget": { + "name": "step" + }, + "link": 146 + } + ], + "outputs": [ + { + "localized_name": "high_sigmas", + "name": "high_sigmas", + "type": "SIGMAS", + "links": [ + 41 + ] + }, + { + "localized_name": "low_sigmas", + "name": "low_sigmas", + "type": "SIGMAS", + "links": [ + 42 + ] + } + ], + "properties": { + "Node name for S&R": "SplitSigmas", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 27, + "type": "KSamplerSelect", + "pos": [ + 3240, + 4640 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 53, + 56 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "res_multistep" + ] + }, + { + "id": 5, + "type": "UNETLoader", + "pos": [ + 890, + 4200 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 171 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 127, + 130 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 11, + "type": "LoraLoaderModelOnly", + "pos": [ + 890, + 4390 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 127 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 174 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 129 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 3 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 12, + "type": "UNETLoader", + "pos": [ + 890, + 4620 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 173 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 34, + 128, + 131 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 16, + "type": "VAEDecode", + "pos": [ + 4410, + 4220 + ], + "size": [ + 250, + 100 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 31 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 99 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 297 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 19, + "type": "SamplerCustom", + "pos": [ + 3580, + 4220 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 19, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 133 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 121 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 123 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 53 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 41 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 125 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 283 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 153 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 40 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true, + 283365432432581, + "randomize", + 1 + ] + }, + { + "id": 18, + "type": "BasicScheduler", + "pos": [ + 3240, + 4230 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 34 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 143 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 15, + "type": "SamplerCustom", + "pos": [ + 3910, + 4220 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 15, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 134 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 122 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 124 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 56 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 42 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 40 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 154 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 31 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + false, + 0, + "fixed", + 1 + ] + }, + { + "id": 50, + "type": "BerniniConditioning", + "pos": [ + 1880, + 4990 + ], + "size": [ + 310, + 380 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 117 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 118 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 119 + }, + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "IMAGE", + "link": 294 + }, + { + "localized_name": "reference_video", + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": 162 + }, + { + "label": "reference_image_0", + "localized_name": "reference_images.reference_image_0", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": 163 + }, + { + "label": "reference_image_1", + "localized_name": "reference_images.reference_image_1", + "name": "reference_images.reference_image_1", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 166 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 167 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 295 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "ref_max_size", + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": 169 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 121, + 122 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 123, + 124 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 125 + ] + } + ], + "properties": { + "Node name for S&R": "BerniniConditioning", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 928, + 1280, + 1, + 1, + 848 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 1670, + 4210 + ], + "size": [ + 710, + 390 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 140 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 117 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 53, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4520 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 131 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 132 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 148 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 134 + ] + } + ], + "title": "Switch (Low Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 54, + "type": "CustomCombo", + "pos": [ + 890, + 3070 + ], + "size": [ + 230, + 350 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "choice", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": 165 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + }, + { + "localized_name": "INDEX", + "name": "INDEX", + "type": "INT", + "links": [ + 135 + ] + } + ], + "properties": { + "Node name for S&R": "CustomCombo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "Default", + 0, + "Default", + "Text to Image", + "Text to Video", + "Image Editing", + "Subject to Image", + "" + ] + }, + { + "id": 57, + "type": "c39e0ea5-b767-460c-b394-b09703772fa6", + "pos": [ + 1400, + 3070 + ], + "size": [ + 390, + 440 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "text_per_line", + "name": "text_per_line", + "type": "STRING", + "widget": { + "name": "text_per_line" + }, + "link": null + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "widget": { + "name": "index" + }, + "link": 135 + } + ], + "outputs": [ + { + "localized_name": "selected_line", + "name": "selected_line", + "type": "STRING", + "links": [ + 137 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "2", + "string" + ], + [ + "56", + "value" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "StringConcatenate", + "pos": [ + 1410, + 3770 + ], + "size": [ + 400, + 250 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "string_a", + "name": "string_a", + "type": "STRING", + "widget": { + "name": "string_a" + }, + "link": 137 + }, + { + "localized_name": "string_b", + "name": "string_b", + "type": "STRING", + "widget": { + "name": "string_b" + }, + "link": 307 + }, + { + "localized_name": "delimiter", + "name": "delimiter", + "type": "STRING", + "widget": { + "name": "delimiter" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 140 + ] + } + ], + "properties": { + "Node name for S&R": "StringConcatenate", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "", + "" + ] + }, + { + "id": 62, + "type": "PrimitiveInt", + "pos": [ + 2460, + 4240 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 141 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 63, + "type": "PrimitiveInt", + "pos": [ + 2470, + 4780 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 142 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 64, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4710 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 141 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 142 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 149 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 143 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 66, + "type": "PrimitiveInt", + "pos": [ + 2460, + 4400 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 144 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 20, + "fixed" + ] + }, + { + "id": 67, + "type": "PrimitiveInt", + "pos": [ + 2470, + 4950 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 145 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 3, + "fixed" + ] + }, + { + "id": 68, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4910 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 144 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 145 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 150 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 146 + ] + } + ], + "title": "Switch (Low Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 72, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 5100 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 151 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 152 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 155 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 153, + 154 + ] + } + ], + "title": "Switch (CFG)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 73, + "type": "PrimitiveFloat", + "pos": [ + 2470, + 5110 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 152 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 71, + "type": "PrimitiveFloat", + "pos": [ + 2460, + 4560 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 151 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 70, + "type": "PrimitiveBoolean", + "pos": [ + 2470, + 5290 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 170 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 147, + 148, + 149, + 150, + 155 + ] + } + ], + "title": "Boolean (Enable Turbo LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 52, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4330 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 130 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 129 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 147 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 133 + ] + } + ], + "title": "Switch (High Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 75, + "type": "MarkdownNote", + "pos": [ + 2450, + 3950 + ], + "size": [ + 340, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can find the original settings here: https://github.com/bytedance/Bernini/blob/main/gradio_demo.py" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 120, + "type": "PrimitiveStringMultiline", + "pos": [ + 890, + 3750 + ], + "size": [ + 470, + 290 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "STRING", + "widget": { + "name": "value" + }, + "link": 308 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 307 + ] + } + ], + "title": "Text Multiline (User Prompt)", + "properties": { + "Node name for S&R": "PrimitiveStringMultiline", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "" + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Prompt & Conditioning", + "bounding": [ + 1650, + 4130, + 760, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Models", + "bounding": [ + 870, + 4130, + 750, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Sampling", + "bounding": [ + 3210, + 4130, + 1160, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Prompt Construction", + "bounding": [ + 870, + 2960, + 960, + 1120 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "System Prompt (Based on task type)", + "bounding": [ + 880, + 3000, + 920, + 680 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Switch Settings", + "bounding": [ + 2440, + 4130, + 740, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Distill LoRA settings", + "bounding": [ + 2450, + 4710, + 310, + 530 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "Original Settings", + "bounding": [ + 2450, + 4170, + 300, + 504 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 128, + "origin_id": 12, + "origin_slot": 0, + "target_id": 29, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 8, + "origin_id": 9, + "origin_slot": 0, + "target_id": 4, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 33, + "origin_id": 18, + "origin_slot": 0, + "target_id": 17, + "target_slot": 0, + "type": "SIGMAS" + }, + { + "id": 146, + "origin_id": 68, + "origin_slot": 0, + "target_id": 17, + "target_slot": 1, + "type": "INT" + }, + { + "id": 127, + "origin_id": 5, + "origin_slot": 0, + "target_id": 11, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 31, + "origin_id": 15, + "origin_slot": 0, + "target_id": 16, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 99, + "origin_id": 7, + "origin_slot": 0, + "target_id": 16, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 133, + "origin_id": 52, + "origin_slot": 0, + "target_id": 19, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 121, + "origin_id": 50, + "origin_slot": 0, + "target_id": 19, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 123, + "origin_id": 50, + "origin_slot": 1, + "target_id": 19, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 53, + "origin_id": 27, + "origin_slot": 0, + "target_id": 19, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 41, + "origin_id": 17, + "origin_slot": 0, + "target_id": 19, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 125, + "origin_id": 50, + "origin_slot": 2, + "target_id": 19, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 153, + "origin_id": 72, + "origin_slot": 0, + "target_id": 19, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 34, + "origin_id": 12, + "origin_slot": 0, + "target_id": 18, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 143, + "origin_id": 64, + "origin_slot": 0, + "target_id": 18, + "target_slot": 2, + "type": "INT" + }, + { + "id": 134, + "origin_id": 53, + "origin_slot": 0, + "target_id": 15, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 122, + "origin_id": 50, + "origin_slot": 0, + "target_id": 15, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 124, + "origin_id": 50, + "origin_slot": 1, + "target_id": 15, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 56, + "origin_id": 27, + "origin_slot": 0, + "target_id": 15, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 42, + "origin_id": 17, + "origin_slot": 1, + "target_id": 15, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 40, + "origin_id": 19, + "origin_slot": 0, + "target_id": 15, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 154, + "origin_id": 72, + "origin_slot": 0, + "target_id": 15, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 117, + "origin_id": 3, + "origin_slot": 0, + "target_id": 50, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 118, + "origin_id": 4, + "origin_slot": 0, + "target_id": 50, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 119, + "origin_id": 7, + "origin_slot": 0, + "target_id": 50, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 7, + "origin_id": 9, + "origin_slot": 0, + "target_id": 3, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 140, + "origin_id": 59, + "origin_slot": 0, + "target_id": 3, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 131, + "origin_id": 12, + "origin_slot": 0, + "target_id": 53, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 132, + "origin_id": 29, + "origin_slot": 0, + "target_id": 53, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 148, + "origin_id": 70, + "origin_slot": 0, + "target_id": 53, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 135, + "origin_id": 54, + "origin_slot": 1, + "target_id": 57, + "target_slot": 1, + "type": "INT" + }, + { + "id": 137, + "origin_id": 57, + "origin_slot": 0, + "target_id": 59, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 141, + "origin_id": 62, + "origin_slot": 0, + "target_id": 64, + "target_slot": 0, + "type": "INT" + }, + { + "id": 142, + "origin_id": 63, + "origin_slot": 0, + "target_id": 64, + "target_slot": 1, + "type": "INT" + }, + { + "id": 149, + "origin_id": 70, + "origin_slot": 0, + "target_id": 64, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 144, + "origin_id": 66, + "origin_slot": 0, + "target_id": 68, + "target_slot": 0, + "type": "INT" + }, + { + "id": 145, + "origin_id": 67, + "origin_slot": 0, + "target_id": 68, + "target_slot": 1, + "type": "INT" + }, + { + "id": 150, + "origin_id": 70, + "origin_slot": 0, + "target_id": 68, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 151, + "origin_id": 71, + "origin_slot": 0, + "target_id": 72, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 152, + "origin_id": 73, + "origin_slot": 0, + "target_id": 72, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 155, + "origin_id": 70, + "origin_slot": 0, + "target_id": 72, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 130, + "origin_id": 5, + "origin_slot": 0, + "target_id": 52, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 129, + "origin_id": 11, + "origin_slot": 0, + "target_id": 52, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 147, + "origin_id": 70, + "origin_slot": 0, + "target_id": 52, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 162, + "origin_id": -10, + "origin_slot": 1, + "target_id": 50, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 163, + "origin_id": -10, + "origin_slot": 2, + "target_id": 50, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 165, + "origin_id": -10, + "origin_slot": 4, + "target_id": 54, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 166, + "origin_id": -10, + "origin_slot": 5, + "target_id": 50, + "target_slot": 7, + "type": "INT" + }, + { + "id": 167, + "origin_id": -10, + "origin_slot": 6, + "target_id": 50, + "target_slot": 8, + "type": "INT" + }, + { + "id": 169, + "origin_id": -10, + "origin_slot": 7, + "target_id": 50, + "target_slot": 11, + "type": "INT" + }, + { + "id": 170, + "origin_id": -10, + "origin_slot": 8, + "target_id": 70, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 171, + "origin_id": -10, + "origin_slot": 9, + "target_id": 5, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 173, + "origin_id": -10, + "origin_slot": 10, + "target_id": 12, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 174, + "origin_id": -10, + "origin_slot": 11, + "target_id": 11, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 175, + "origin_id": -10, + "origin_slot": 11, + "target_id": 29, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 176, + "origin_id": -10, + "origin_slot": 12, + "target_id": 9, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 177, + "origin_id": -10, + "origin_slot": 13, + "target_id": 7, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 283, + "origin_id": -10, + "origin_slot": 14, + "target_id": 19, + "target_slot": 7, + "type": "INT" + }, + { + "id": 294, + "origin_id": -10, + "origin_slot": 0, + "target_id": 50, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 295, + "origin_id": -10, + "origin_slot": 15, + "target_id": 50, + "target_slot": 9, + "type": "INT" + }, + { + "id": 297, + "origin_id": 16, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 307, + "origin_id": 120, + "origin_slot": 0, + "target_id": 59, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 308, + "origin_id": -10, + "origin_slot": 3, + "target_id": 120, + "target_slot": 0, + "type": "STRING" + } + ], + "extra": {}, + "category": "Image generation and editing/Edit image", + "description": "Edits a single image using a text prompt, leveraging Bernini-R's latent semantic planning for changes like object addition, removal, or style transfer. Ideal for creative edits requiring precise semantic understanding, such as adding a snowman to a scene or altering an object's appearance." + }, + { + "id": "c39e0ea5-b767-460c-b394-b09703772fa6", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 157, + "lastLinkId": 308, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Select Per-Line Text by Index", + "description": "Selects one line from multiline text by zero-based index for batch or list-driven prompt workflows.", + "inputNode": { + "id": -10, + "bounding": [ + -990, + 8595, + 128, + 88 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 710, + 8585, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "75417d82-a934-4ac9-b667-d8dcd5a3bfb3", + "name": "text_per_line", + "type": "STRING", + "linkIds": [ + 13 + ], + "localized_name": "text_per_line", + "pos": [ + -886, + 8619 + ] + }, + { + "id": "46e69a73-1804-4ca6-9175-31445bf0be96", + "name": "index", + "type": "INT", + "linkIds": [ + 14 + ], + "localized_name": "index", + "pos": [ + -886, + 8639 + ] + } + ], + "outputs": [ + { + "id": "e34e8ad1-84d2-4bd2-a460-eb7de6067c10", + "name": "selected_line", + "type": "STRING", + "linkIds": [ + 10 + ], + "localized_name": "selected_line", + "pos": [ + 734, + 8609 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 1, + "type": "PreviewAny", + "pos": [ + -500, + 8400 + ], + "size": [ + 230, + 180 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "PreviewAny", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 2, + "type": "RegexExtract", + "pos": [ + -240, + 8740 + ], + "size": [ + 470, + 460 + ], + "flags": {}, + "order": 1, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": 13 + }, + { + "localized_name": "regex_pattern", + "name": "regex_pattern", + "type": "STRING", + "widget": { + "name": "regex_pattern" + }, + "link": 9 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "case_insensitive", + "name": "case_insensitive", + "type": "BOOLEAN", + "widget": { + "name": "case_insensitive" + }, + "link": null + }, + { + "localized_name": "multiline", + "name": "multiline", + "type": "BOOLEAN", + "widget": { + "name": "multiline" + }, + "link": null + }, + { + "localized_name": "dotall", + "name": "dotall", + "type": "BOOLEAN", + "widget": { + "name": "dotall" + }, + "link": null + }, + { + "localized_name": "group_index", + "name": "group_index", + "type": "INT", + "widget": { + "name": "group_index" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "RegexExtract", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "You are a helpful assistant.\nYou are a helpful assistant specialized in text-to-image generation.\nYou are a helpful assistant specialized in text-to-video generation.\nYou are a helpful assistant specialized in image editing.\nYou are a helpful assistant specialized in subject-to-image generation.", + "", + "First Group", + false, + false, + false, + 1 + ] + }, + { + "id": 56, + "type": "PrimitiveInt", + "pos": [ + -810, + 8400 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 14 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 1 + ] + } + ], + "title": "Int (line index)", + "properties": { + "Node name for S&R": "Int (line index)", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 0, + "fixed" + ] + }, + { + "id": 8, + "type": "StringReplace", + "pos": [ + -240, + 8400 + ], + "size": [ + 400, + 280 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": null + }, + { + "localized_name": "find", + "name": "find", + "type": "STRING", + "widget": { + "name": "find" + }, + "link": null + }, + { + "localized_name": "replace", + "name": "replace", + "type": "STRING", + "widget": { + "name": "replace" + }, + "link": 6 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 9 + ] + } + ], + "properties": { + "Node name for S&R": "StringReplace", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "^(?:[^\\n]*\\n){index}([^\\n]*)(?:\\n|$)", + "index", + "" + ] + } + ], + "groups": [], + "links": [ + { + "id": 1, + "origin_id": 56, + "origin_slot": 0, + "target_id": 1, + "target_slot": 0, + "type": "INT" + }, + { + "id": 9, + "origin_id": 8, + "origin_slot": 0, + "target_id": 2, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 6, + "origin_id": 1, + "origin_slot": 0, + "target_id": 8, + "target_slot": 2, + "type": "STRING" + }, + { + "id": 10, + "origin_id": 2, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 13, + "origin_id": -10, + "origin_slot": 0, + "target_id": 2, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 14, + "origin_id": -10, + "origin_slot": 1, + "target_id": 56, + "target_slot": 0, + "type": "INT" + } + ], + "extra": { + "ue_links": [], + "links_added_by_ue": [] + } + } + ] + }, + "extra": { + "BlueprintDescription": "Edits a single image using a text prompt, leveraging Bernini-R's latent semantic planning for changes like object addition, removal, or style transfer. Ideal for creative edits requiring precise semantic understanding, such as adding a snowman to a scene or altering an object's appearance." + } +} \ No newline at end of file diff --git a/blueprints/Image to Gaussian Splat (TripoSplat).json b/blueprints/Image to Gaussian Splat (TripoSplat).json new file mode 100644 index 000000000..13da12be3 --- /dev/null +++ b/blueprints/Image to Gaussian Splat (TripoSplat).json @@ -0,0 +1,1983 @@ +{ + "revision": 0, + "last_node_id": 118, + "last_link_id": 0, + "nodes": [ + { + "id": 118, + "type": "6c0b94f2-a3d2-4056-93f7-4072126afe39", + "pos": [ + 790, + 1900 + ], + "size": [ + 430, + 670 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "label": "auto_remove_background", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": null + }, + { + "label": "num_gaussians", + "name": "num_gaussians_1", + "type": "INT", + "widget": { + "name": "num_gaussians_1" + }, + "link": null + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "label": "splat_vae", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "flux2_vae", + "name": "vae_name_1", + "type": "COMBO", + "widget": { + "name": "vae_name_1" + }, + "link": null + }, + { + "label": "bg_removal_model", + "name": "bg_removal_name", + "type": "COMBO", + "widget": { + "name": "bg_removal_name" + }, + "link": null + }, + { + "label": "mask", + "name": "on_false", + "type": "MASK", + "link": null + }, + { + "label": "enable_preview", + "name": "switch_1", + "type": "BOOLEAN", + "widget": { + "name": "switch_1" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "splat", + "name": "splat", + "type": "SPLAT", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "103", + "switch" + ], + [ + "108", + "num_gaussians" + ], + [ + "102", + "seed" + ], + [ + "111", + "unet_name" + ], + [ + "105", + "clip_name" + ], + [ + "106", + "vae_name" + ], + [ + "107", + "vae_name" + ], + [ + "109", + "bg_removal_name" + ], + [ + "112", + "switch" + ], + [ + "101", + "$$canvas-image-preview" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "title": "Image to Gaussian Splat (TripoSplat)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "6c0b94f2-a3d2-4056-93f7-4072126afe39", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 118, + "lastLinkId": 219, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Image to Gaussian Splat (TripoSplat)", + "inputNode": { + "id": -10, + "bounding": [ + -110, + 1050, + 194.666015625, + 268 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 2250, + 700, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "736cc4fd-77da-4fb7-8db7-d94296654571", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 165, + 195 + ], + "localized_name": "image", + "pos": [ + 60.666015625, + 1074 + ] + }, + { + "id": "bc100771-6c6a-4223-a6b1-62f8c7b1c5b6", + "name": "switch", + "type": "BOOLEAN", + "linkIds": [ + 184 + ], + "label": "auto_remove_background", + "pos": [ + 60.666015625, + 1094 + ] + }, + { + "id": "13a86171-bf5d-4328-ae51-116502beb274", + "name": "num_gaussians_1", + "type": "INT", + "linkIds": [ + 196 + ], + "label": "num_gaussians", + "pos": [ + 60.666015625, + 1114 + ] + }, + { + "id": "38ebeca8-4674-4a73-a686-f8b9444c7bb6", + "name": "seed", + "type": "INT", + "linkIds": [ + 197 + ], + "pos": [ + 60.666015625, + 1134 + ] + }, + { + "id": "6952068a-04ba-4e56-b702-0986f69b0d96", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 201 + ], + "pos": [ + 60.666015625, + 1154 + ] + }, + { + "id": "206819d5-2ecb-4788-8dcf-134138e46bbf", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 202 + ], + "pos": [ + 60.666015625, + 1174 + ] + }, + { + "id": "83a8d3d1-4c75-4ce0-a03c-face70f57ac9", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 203 + ], + "label": "splat_vae", + "pos": [ + 60.666015625, + 1194 + ] + }, + { + "id": "dce14e7e-37ca-493b-8dbe-24333d7cee63", + "name": "vae_name_1", + "type": "COMBO", + "linkIds": [ + 204 + ], + "label": "flux2_vae", + "pos": [ + 60.666015625, + 1214 + ] + }, + { + "id": "d9c843e7-567b-4acc-813d-752579cb89b5", + "name": "bg_removal_name", + "type": "COMBO", + "linkIds": [ + 205 + ], + "label": "bg_removal_model", + "pos": [ + 60.666015625, + 1234 + ] + }, + { + "id": "07d672fb-6e35-44d8-b973-7577c3468208", + "name": "on_false", + "type": "MASK", + "linkIds": [ + 209 + ], + "label": "mask", + "pos": [ + 60.666015625, + 1254 + ] + }, + { + "id": "8ae4f3b4-e3d2-4d42-84fa-4e9af9ffc3e7", + "name": "switch_1", + "type": "BOOLEAN", + "linkIds": [ + 212 + ], + "label": "enable_preview", + "pos": [ + 60.666015625, + 1274 + ] + } + ], + "outputs": [ + { + "id": "4185ff5d-4179-4938-9fb6-cea3a8322606", + "name": "splat", + "type": "SPLAT", + "linkIds": [ + 156, + 156 + ], + "localized_name": "splat", + "pos": [ + 2274, + 724 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 100, + "type": "TripoSplatConditioning", + "pos": [ + 940, + 800 + ], + "size": [ + 290, + 120 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 45 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 46 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 47 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 48 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 49 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "TripoSplatConditioning", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 101, + "type": "PreviewImage", + "pos": [ + 1360, + 1290 + ], + "size": [ + 440, + 530 + ], + "flags": {}, + "order": 1, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 38 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 102, + "type": "KSampler", + "pos": [ + 1350, + 570 + ], + "size": [ + 290, + 590 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 211 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 48 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 49 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 54 + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 197 + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": null + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + }, + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 103 + ] + } + ], + "properties": { + "Node name for S&R": "KSampler", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 46, + "fixed", + 20, + 3, + "dpmpp_2m", + "simple", + 1 + ] + }, + { + "id": 103, + "type": "ComfySwitchNode", + "pos": [ + 1020, + 1330 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 208 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 170 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 184 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 95 + ] + } + ], + "title": "Switch: Mask Source", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true + ] + }, + { + "id": 104, + "type": "TripoSplatPreprocessImage", + "pos": [ + 940, + 990 + ], + "size": [ + 300, + 160 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 195 + }, + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 95 + }, + { + "localized_name": "erode_radius", + "name": "erode_radius", + "type": "INT", + "widget": { + "name": "erode_radius" + }, + "link": null + }, + { + "localized_name": "size", + "name": "size", + "type": "INT", + "widget": { + "name": "size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "links": [ + 38, + 47 + ] + } + ], + "properties": { + "Node name for S&R": "TripoSplatPreprocessImage", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1, + 1024 + ] + }, + { + "id": 105, + "type": "CLIPVisionLoader", + "pos": [ + 410, + 760 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 202 + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 45 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "dino_v3_vit_h.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/clip_vision/dino_v3_vit_h.safetensors", + "directory": "clip_vision" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "dino_v3_vit_h.safetensors" + ] + }, + { + "id": 106, + "type": "VAELoader", + "pos": [ + 420, + 930 + ], + "size": [ + 410, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 203 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 104, + 216 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "triposplat_vae_decoder_fp16.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/vae/triposplat_vae_decoder_fp16.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "triposplat_vae_decoder_fp16.safetensors" + ] + }, + { + "id": 107, + "type": "VAELoader", + "pos": [ + 410, + 1120 + ], + "size": [ + 420, + 110 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 204 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 46 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/vae/flux2-vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "flux2-vae.safetensors" + ] + }, + { + "id": 108, + "type": "VAEDecodeTripoSplat", + "pos": [ + 1730, + 570 + ], + "size": [ + 430, + 160 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 103 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 104 + }, + { + "localized_name": "num_gaussians", + "name": "num_gaussians", + "type": "INT", + "widget": { + "name": "num_gaussians" + }, + "link": 196 + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "splat", + "name": "splat", + "type": "SPLAT", + "links": [ + 156 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecodeTripoSplat", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 262144, + 790219963981395, + "fixed" + ] + }, + { + "id": 109, + "type": "e527b93c-76f7-485d-b285-fcf78914a4d5", + "pos": [ + 410, + 1330 + ], + "size": [ + 350, + 160 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 165 + }, + { + "name": "bg_removal_name", + "type": "COMBO", + "widget": { + "name": "bg_removal_name" + }, + "link": 205 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 166, + 194 + ] + }, + { + "name": "mask", + "type": "MASK", + "links": [ + 170 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "115", + "bg_removal_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + }, + { + "id": 110, + "type": "InvertMask", + "pos": [ + 430, + 1540 + ], + "size": [ + 230, + 80 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 209 + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 208 + ] + } + ], + "properties": { + "Node name for S&R": "InvertMask", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 111, + "type": "UNETLoader", + "pos": [ + 410, + 560 + ], + "size": [ + 410, + 140 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 201 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 213, + 215 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "triposplat_fp16.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/diffusion_models/triposplat_fp16.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "triposplat_fp16.safetensors", + "default" + ] + }, + { + "id": 112, + "type": "ComfySwitchNode", + "pos": [ + 930, + 370 + ], + "size": [ + 300, + 140 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 213 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 217 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 212 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 211 + ] + } + ], + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true + ] + }, + { + "id": 113, + "type": "TripoSplatSamplingPreview", + "pos": [ + 940, + 560 + ], + "size": [ + 290, + 190 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 215 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 216 + }, + { + "localized_name": "octree_level", + "name": "octree_level", + "type": "INT", + "widget": { + "name": "octree_level" + }, + "link": null + }, + { + "localized_name": "num_gaussians", + "name": "num_gaussians", + "type": "INT", + "widget": { + "name": "num_gaussians" + }, + "link": null + }, + { + "localized_name": "yaw", + "name": "yaw", + "type": "FLOAT", + "widget": { + "name": "yaw" + }, + "link": null + }, + { + "localized_name": "pitch", + "name": "pitch", + "type": "FLOAT", + "widget": { + "name": "pitch" + }, + "link": null + }, + { + "localized_name": "point_size", + "name": "point_size", + "type": "INT", + "widget": { + "name": "point_size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 217 + ] + } + ], + "properties": { + "Node name for S&R": "TripoSplatSamplingPreview", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 5, + 16384, + 90, + 15, + 2 + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 400, + 490, + 440, + 748.625 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Image Preprocessing", + "bounding": [ + 400, + 1260, + 910, + 370 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 45, + "origin_id": 105, + "origin_slot": 0, + "target_id": 100, + "target_slot": 0, + "type": "CLIP_VISION" + }, + { + "id": 46, + "origin_id": 107, + "origin_slot": 0, + "target_id": 100, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 47, + "origin_id": 104, + "origin_slot": 0, + "target_id": 100, + "target_slot": 2, + "type": "IMAGE" + }, + { + "id": 38, + "origin_id": 104, + "origin_slot": 0, + "target_id": 101, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 103, + "origin_id": 102, + "origin_slot": 0, + "target_id": 108, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 104, + "origin_id": 106, + "origin_slot": 0, + "target_id": 108, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 48, + "origin_id": 100, + "origin_slot": 0, + "target_id": 102, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 49, + "origin_id": 100, + "origin_slot": 1, + "target_id": 102, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 54, + "origin_id": 100, + "origin_slot": 2, + "target_id": 102, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 170, + "origin_id": 109, + "origin_slot": 1, + "target_id": 103, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 95, + "origin_id": 103, + "origin_slot": 0, + "target_id": 104, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 165, + "origin_id": -10, + "origin_slot": 0, + "target_id": 109, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 156, + "origin_id": 108, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "SPLAT" + }, + { + "id": 184, + "origin_id": -10, + "origin_slot": 1, + "target_id": 103, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 195, + "origin_id": -10, + "origin_slot": 0, + "target_id": 104, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 196, + "origin_id": -10, + "origin_slot": 2, + "target_id": 108, + "target_slot": 2, + "type": "INT" + }, + { + "id": 197, + "origin_id": -10, + "origin_slot": 3, + "target_id": 102, + "target_slot": 4, + "type": "INT" + }, + { + "id": 201, + "origin_id": -10, + "origin_slot": 4, + "target_id": 111, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 202, + "origin_id": -10, + "origin_slot": 5, + "target_id": 105, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 203, + "origin_id": -10, + "origin_slot": 6, + "target_id": 106, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 204, + "origin_id": -10, + "origin_slot": 7, + "target_id": 107, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 205, + "origin_id": -10, + "origin_slot": 8, + "target_id": 109, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 208, + "origin_id": 110, + "origin_slot": 0, + "target_id": 103, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 209, + "origin_id": -10, + "origin_slot": 9, + "target_id": 110, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 211, + "origin_id": 112, + "origin_slot": 0, + "target_id": 102, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 212, + "origin_id": -10, + "origin_slot": 10, + "target_id": 112, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 213, + "origin_id": 111, + "origin_slot": 0, + "target_id": 112, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 215, + "origin_id": 111, + "origin_slot": 0, + "target_id": 113, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 216, + "origin_id": 106, + "origin_slot": 0, + "target_id": 113, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 217, + "origin_id": 113, + "origin_slot": 0, + "target_id": 112, + "target_slot": 1, + "type": "MODEL" + } + ], + "extra": {}, + "category": "3D", + "description": "This subgraph takes a single 2D image as input and generates a variable number of 3D Gaussians (up to 262,144) as output, enabling high-quality 3D reconstruction. It is ideal for asset creation, AR/VR, game development, and simulation environments, handling diverse image styles from photos to illustrations." + }, + { + "id": "e527b93c-76f7-485d-b285-fcf78914a4d5", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 118, + "lastLinkId": 219, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Remove Background (BiRefNet)", + "description": "Removes or replaces image backgrounds using BiRefNet segmentation and alpha compositing.", + "inputNode": { + "id": -10, + "bounding": [ + -6810, + 1480, + 150.9140625, + 88 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + -6169.049695722246, + 1475.2619799128663, + 128, + 88 + ] + }, + "inputs": [ + { + "id": "7bc321cd-df31-4c39-aaf7-7f0d01326189", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 5, + 7 + ], + "localized_name": "image", + "pos": [ + -6683.0859375, + 1504 + ] + }, + { + "id": "e89d2cd8-daa3-4e29-8a69-851db85072cb", + "name": "bg_removal_name", + "type": "COMBO", + "linkIds": [ + 12 + ], + "pos": [ + -6683.0859375, + 1524 + ] + } + ], + "outputs": [ + { + "id": "16e7863c-4c38-46c2-aa74-e82991fbfe8d", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 8 + ], + "localized_name": "IMAGE", + "pos": [ + -6145.049695722246, + 1499.2619799128663 + ] + }, + { + "id": "f7240c19-5b80-406e-a8e2-9b12440ee2d6", + "name": "mask", + "type": "MASK", + "linkIds": [ + 11 + ], + "pos": [ + -6145.049695722246, + 1519.2619799128663 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 114, + "type": "RemoveBackground", + "pos": [ + -6540, + 1440 + ], + "size": [ + 310, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "bg_removal_model", + "name": "bg_removal_model", + "type": "BACKGROUND_REMOVAL", + "link": 3 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 5 + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 4, + 11 + ] + } + ], + "properties": { + "Node name for S&R": "RemoveBackground", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 115, + "type": "LoadBackgroundRemovalModel", + "pos": [ + -6540, + 1300 + ], + "size": [ + 320, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "bg_removal_name", + "name": "bg_removal_name", + "type": "COMBO", + "widget": { + "name": "bg_removal_name" + }, + "link": 12 + } + ], + "outputs": [ + { + "localized_name": "bg_model", + "name": "bg_model", + "type": "BACKGROUND_REMOVAL", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "LoadBackgroundRemovalModel", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "birefnet.safetensors", + "url": "https://huggingface.co/Comfy-Org/BiRefNet/resolve/main/background_removal/birefnet.safetensors", + "directory": "background_removal" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "birefnet.safetensors" + ] + }, + { + "id": 116, + "type": "InvertMask", + "pos": [ + -6530, + 1570 + ], + "size": [ + 290, + 80 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 4 + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "InvertMask", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 117, + "type": "JoinImageWithAlpha", + "pos": [ + -6530, + 1670 + ], + "size": [ + 290, + 100 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 7 + }, + { + "localized_name": "alpha", + "name": "alpha", + "type": "MASK", + "link": 6 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 8 + ] + } + ], + "properties": { + "Node name for S&R": "JoinImageWithAlpha", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + } + ], + "groups": [], + "links": [ + { + "id": 3, + "origin_id": 115, + "origin_slot": 0, + "target_id": 114, + "target_slot": 0, + "type": "BACKGROUND_REMOVAL" + }, + { + "id": 4, + "origin_id": 114, + "origin_slot": 0, + "target_id": 116, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 6, + "origin_id": 116, + "origin_slot": 0, + "target_id": 117, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 5, + "origin_id": -10, + "origin_slot": 0, + "target_id": 114, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 7, + "origin_id": -10, + "origin_slot": 0, + "target_id": 117, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 8, + "origin_id": 117, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 11, + "origin_id": 114, + "origin_slot": 0, + "target_id": -20, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 12, + "origin_id": -10, + "origin_slot": 1, + "target_id": 115, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {} + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph takes a single 2D image as input and generates a variable number of 3D Gaussians (up to 262,144) as output, enabling high-quality 3D reconstruction. It is ideal for asset creation, AR/VR, game development, and simulation environments, handling diverse image styles from photos to illustrations." + } +} \ No newline at end of file diff --git a/blueprints/Text to Image (Anima Base 1.0).json b/blueprints/Text to Image (Anima Base 1.0).json new file mode 100644 index 000000000..379e7c605 --- /dev/null +++ b/blueprints/Text to Image (Anima Base 1.0).json @@ -0,0 +1,1088 @@ +{ + "revision": 0, + "last_node_id": 60, + "last_link_id": 0, + "nodes": [ + { + "id": 60, + "type": "a3c0dab6-b250-4585-a0f9-8fb8b074fb2f", + "pos": [ + -10, + 130 + ], + "size": [ + 500, + 640 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": null + }, + { + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "title": "Text to Image (Anima Base 1.0)", + "properties": { + "proxyWidgets": [ + [ + "11", + "text" + ], + [ + "28", + "width" + ], + [ + "28", + "height" + ], + [ + "19", + "steps" + ], + [ + "19", + "cfg" + ], + [ + "19", + "seed" + ], + [ + "44", + "unet_name" + ], + [ + "45", + "clip_name" + ], + [ + "15", + "vae_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.18.1", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "a3c0dab6-b250-4585-a0f9-8fb8b074fb2f", + "version": 1, + "state": { + "lastGroupId": 3, + "lastNodeId": 70, + "lastLinkId": 104, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Text to Image (Anima Base 1.0)", + "inputNode": { + "id": -10, + "bounding": [ + -330, + 530, + 120, + 220 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1229.9999873482075, + 505, + 120, + 60 + ] + }, + "inputs": [ + { + "id": "4693f350-6ba0-446d-80d4-3038c661d26c", + "name": "text", + "type": "STRING", + "linkIds": [ + 95 + ], + "label": "prompt", + "pos": [ + -230, + 550 + ] + }, + { + "id": "4a7886a9-4ed7-49bb-afc2-977bb78a303d", + "name": "width", + "type": "INT", + "linkIds": [ + 96 + ], + "pos": [ + -230, + 570 + ] + }, + { + "id": "f6c04461-d29e-49e3-8790-07bb662bbbfe", + "name": "height", + "type": "INT", + "linkIds": [ + 97 + ], + "pos": [ + -230, + 590 + ] + }, + { + "id": "7a24f998-3808-4837-8bff-52304ad09fb6", + "name": "steps", + "type": "INT", + "linkIds": [ + 98 + ], + "pos": [ + -230, + 610 + ] + }, + { + "id": "aaa99698-b222-40fe-b946-28067528a85c", + "name": "cfg", + "type": "FLOAT", + "linkIds": [ + 99 + ], + "pos": [ + -230, + 630 + ] + }, + { + "id": "053df9ae-7311-4816-aa23-7fa13c656ced", + "name": "seed", + "type": "INT", + "linkIds": [ + 100 + ], + "pos": [ + -230, + 650 + ] + }, + { + "id": "c59194ea-015c-41a7-8edd-ae7ffc220b63", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 101 + ], + "pos": [ + -230, + 670 + ] + }, + { + "id": "e655aa3b-2db7-4e25-9ea2-61550fa7ae2d", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 102 + ], + "pos": [ + -230, + 690 + ] + }, + { + "id": "94965a7a-74dd-4f5a-87e3-9f87995d554f", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 103 + ], + "pos": [ + -230, + 710 + ] + } + ], + "outputs": [ + { + "id": "ef85ac0a-2152-4232-bfa1-929cfc913718", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 82 + ], + "localized_name": "IMAGE", + "pos": [ + 1249.9999873482075, + 525 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 45, + "type": "CLIPLoader", + "pos": [ + -60, + 380 + ], + "size": [ + 310, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 102 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 80, + 81 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.11.0", + "models": [ + { + "name": "qwen_3_06b_base.safetensors", + "url": "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/text_encoders/qwen_3_06b_base.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen_3_06b_base.safetensors", + "stable_diffusion", + "default" + ] + }, + { + "id": 15, + "type": "VAELoader", + "pos": [ + -50, + 610 + ], + "size": [ + 310, + 100 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 103 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 11 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 880, + 840 + ], + "size": [ + 230, + 90 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 10 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 11 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 28, + "type": "EmptyLatentImage", + "pos": [ + -50, + 830 + ], + "size": [ + 310, + 150 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 96 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 97 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 12, + "type": "CLIPTextEncode", + "pos": [ + 330, + 830 + ], + "size": [ + 490, + 140 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 81 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 40 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.65", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "worst quality, low quality, score_1, score_2, score_3, blurry, jpeg artifacts, sepia" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 19, + "type": "KSampler", + "pos": [ + 870, + 120 + ], + "size": [ + 300, + 620 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 79 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 39 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 40 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 78 + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 100 + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 98 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 99 + }, + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "KSampler", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 875817230929465, + "fixed", + 30, + 4, + "er_sde", + "simple", + 1 + ] + }, + { + "id": 11, + "type": "CLIPTextEncode", + "pos": [ + 320, + 170 + ], + "size": [ + 490, + 610 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 80 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 95 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 39 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.65", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 44, + "type": "UNETLoader", + "pos": [ + -50, + 170 + ], + "size": [ + 310, + 130 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 101 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.11.0", + "models": [ + { + "name": "anima-base-v1.0.safetensors", + "url": "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/diffusion_models/anima-base-v1.0.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "anima-base-v1.0.safetensors", + "default" + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Model", + "bounding": [ + -80, + 80, + 360, + 640 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Image Size(1MP)", + "bounding": [ + -80, + 750, + 360, + 240 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Prompt", + "bounding": [ + 300, + 80, + 530, + 910 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 10, + "origin_id": 19, + "origin_slot": 0, + "target_id": 8, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 11, + "origin_id": 15, + "origin_slot": 0, + "target_id": 8, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 81, + "origin_id": 45, + "origin_slot": 0, + "target_id": 12, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 79, + "origin_id": 44, + "origin_slot": 0, + "target_id": 19, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 39, + "origin_id": 11, + "origin_slot": 0, + "target_id": 19, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 40, + "origin_id": 12, + "origin_slot": 0, + "target_id": 19, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 78, + "origin_id": 28, + "origin_slot": 0, + "target_id": 19, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 80, + "origin_id": 45, + "origin_slot": 0, + "target_id": 11, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 82, + "origin_id": 8, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 95, + "origin_id": -10, + "origin_slot": 0, + "target_id": 11, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 96, + "origin_id": -10, + "origin_slot": 1, + "target_id": 28, + "target_slot": 0, + "type": "INT" + }, + { + "id": 97, + "origin_id": -10, + "origin_slot": 2, + "target_id": 28, + "target_slot": 1, + "type": "INT" + }, + { + "id": 98, + "origin_id": -10, + "origin_slot": 3, + "target_id": 19, + "target_slot": 5, + "type": "INT" + }, + { + "id": 99, + "origin_id": -10, + "origin_slot": 4, + "target_id": 19, + "target_slot": 6, + "type": "FLOAT" + }, + { + "id": 100, + "origin_id": -10, + "origin_slot": 5, + "target_id": 19, + "target_slot": 4, + "type": "INT" + }, + { + "id": 101, + "origin_id": -10, + "origin_slot": 6, + "target_id": 44, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 102, + "origin_id": -10, + "origin_slot": 7, + "target_id": 45, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 103, + "origin_id": -10, + "origin_slot": 8, + "target_id": 15, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Image generation and editing/Text to image", + "description": "This subgraph generates non-photorealistic illustrations from text prompts using a 2-billion-parameter model optimized for anime concepts, characters, and styles. It is ideal for creating artistic images, concept art, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph generates non-photorealistic illustrations from text prompts using a 2-billion-parameter model optimized for anime concepts, characters, and styles. It is ideal for creating artistic images, concept art, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." + } +} \ No newline at end of file diff --git a/blueprints/Text to Image (Anima).json b/blueprints/Text to Image (Anima).json index 787908ca9..dcf6e5973 100644 --- a/blueprints/Text to Image (Anima).json +++ b/blueprints/Text to Image (Anima).json @@ -1077,9 +1077,12 @@ } ], "extra": {}, - "category": "Image generation and editing/Text to image" + "category": "Image generation and editing/Text to image", + "description": "This subgraph converts text prompts into non-photorealistic illustrations using a 2-billion-parameter model optimized for anime and artistic styles. It is ideal for generating concept art, character designs, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." } ] }, - "extra": {} + "extra": { + "BlueprintDescription": "This subgraph converts text prompts into non-photorealistic illustrations using a 2-billion-parameter model optimized for anime and artistic styles. It is ideal for generating concept art, character designs, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." + } } \ No newline at end of file diff --git a/blueprints/Text to Image (Ideogram v4).json b/blueprints/Text to Image (Ideogram v4).json new file mode 100644 index 000000000..0d5c1d7c7 --- /dev/null +++ b/blueprints/Text to Image (Ideogram v4).json @@ -0,0 +1,2473 @@ +{ + "revision": 0, + "last_node_id": 204, + "last_link_id": 0, + "nodes": [ + { + "id": 204, + "type": "3aa9dcf6-e101-4b91-abee-6d7a7c10023b", + "pos": [ + 6380, + 1410 + ], + "size": [ + 440, + 690 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "width", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "height", + "name": "value_1", + "type": "INT", + "widget": { + "name": "value_1" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "unconditional_unet", + "name": "unet_name_1", + "type": "COMBO", + "widget": { + "name": "unet_name_1" + }, + "link": null + }, + { + "label": "mode", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "186", + "text" + ], + [ + "188", + "value" + ], + [ + "189", + "value" + ], + [ + "184", + "noise_seed" + ], + [ + "185", + "unet_name" + ], + [ + "187", + "clip_name" + ], + [ + "177", + "vae_name" + ], + [ + "200", + "unet_name" + ], + [ + "202", + "choice" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "title": "Text to Image (Ideogram v4)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "3aa9dcf6-e101-4b91-abee-6d7a7c10023b", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 204, + "lastLinkId": 252, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Text to Image (Ideogram v4)", + "inputNode": { + "id": -10, + "bounding": [ + 3490, + 920, + 154.921875, + 228 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 6850, + 936, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "4bc742d1-7b4b-452c-90d9-0d76ebcdae76", + "name": "text", + "type": "STRING", + "linkIds": [ + 152 + ], + "label": "prompt", + "pos": [ + 3620.921875, + 944 + ] + }, + { + "id": "8d4038eb-73c7-45e9-bba1-f068f55e8d32", + "name": "value", + "type": "INT", + "linkIds": [ + 153 + ], + "label": "width", + "pos": [ + 3620.921875, + 964 + ] + }, + { + "id": "281550e6-6acf-4cbe-aec1-9eb803b4dec1", + "name": "value_1", + "type": "INT", + "linkIds": [ + 154 + ], + "label": "height", + "pos": [ + 3620.921875, + 984 + ] + }, + { + "id": "fae56884-2f1a-470b-a25f-40e7a87ef69d", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 155 + ], + "pos": [ + 3620.921875, + 1004 + ] + }, + { + "id": "3497309c-a7d7-4e28-9330-142c15881632", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 156 + ], + "pos": [ + 3620.921875, + 1024 + ] + }, + { + "id": "e87126db-7147-465e-b129-370ed2c6cc22", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 157 + ], + "pos": [ + 3620.921875, + 1044 + ] + }, + { + "id": "a1e6c080-b11b-4d5c-a3a8-fcf4df654cf7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 158 + ], + "pos": [ + 3620.921875, + 1064 + ] + }, + { + "id": "b0d16516-95de-44d9-bea8-3cd2e7c78e9a", + "name": "unet_name_1", + "type": "COMBO", + "linkIds": [ + 216 + ], + "label": "unconditional_unet", + "pos": [ + 3620.921875, + 1084 + ] + }, + { + "id": "249fd825-e6b3-489d-a341-6d8050500f5e", + "name": "choice", + "type": "COMBO", + "linkIds": [ + 219 + ], + "label": "mode", + "pos": [ + 3620.921875, + 1104 + ] + } + ], + "outputs": [ + { + "id": "b81e4f60-e543-4f02-875b-b0f1bdc274f2", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 25 + ], + "localized_name": "IMAGE", + "pos": [ + 6874, + 960 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 177, + "type": "VAELoader", + "pos": [ + 4730, + 1220 + ], + "size": [ + 470, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 158 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 17 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "flux2-vae.safetensors" + ] + }, + { + "id": 178, + "type": "ConditioningZeroOut", + "pos": [ + 5450, + 960 + ], + "size": [ + 250, + 80 + ], + "flags": { + "collapsed": false + }, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "conditioning", + "name": "conditioning", + "type": "CONDITIONING", + "link": 8 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 214 + ] + } + ], + "properties": { + "Node name for S&R": "ConditioningZeroOut", + "cnr_id": "comfy-core", + "ver": "0.9.1", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 179, + "type": "EmptyFlux2LatentImage", + "pos": [ + 5330, + 1180 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 33 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 36 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 15 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyFlux2LatentImage", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 180, + "type": "SamplerCustomAdvanced", + "pos": [ + 6160, + 500 + ], + "size": [ + 290, + 170 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "noise", + "name": "noise", + "type": "NOISE", + "link": 11 + }, + { + "localized_name": "guider", + "name": "guider", + "type": "GUIDER", + "link": 215 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 13 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 14 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 15 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 16 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustomAdvanced", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 181, + "type": "VAEDecode", + "pos": [ + 6560, + 500 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 16 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 25 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 182, + "type": "KSamplerSelect", + "pos": [ + 5790, + 1100 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 13 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 183, + "type": "Ideogram4Scheduler", + "pos": [ + 5790, + 1260 + ], + "size": [ + 270, + 240 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 207 + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 34 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 37 + }, + { + "localized_name": "mu", + "name": "mu", + "type": "FLOAT", + "widget": { + "name": "mu" + }, + "link": 208 + }, + { + "localized_name": "std", + "name": "std", + "type": "FLOAT", + "widget": { + "name": "std" + }, + "link": 209 + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 14 + ] + } + ], + "properties": { + "Node name for S&R": "Ideogram4Scheduler", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 20, + 1024, + 1024, + 0.5, + 1.75 + ] + }, + { + "id": 184, + "type": "RandomNoise", + "pos": [ + 5780, + 490 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 155 + } + ], + "outputs": [ + { + "localized_name": "NOISE", + "name": "NOISE", + "type": "NOISE", + "links": [ + 11 + ] + } + ], + "properties": { + "Node name for S&R": "RandomNoise", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 885894517601261, + "randomize" + ] + }, + { + "id": 185, + "type": "UNETLoader", + "pos": [ + 4720, + 520 + ], + "size": [ + 470, + 170 + ], + "flags": {}, + "order": 8, + "mode": 0, + "showAdvanced": true, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 156 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 222 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "ideogram4_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Ideogram-4/resolve/main/diffusion_models/ideogram4_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "ideogram4_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 186, + "type": "CLIPTextEncode", + "pos": [ + 5270, + 500 + ], + "size": [ + 430, + 420 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 24 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 152 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 8, + 213 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 187, + "type": "CLIPLoader", + "pos": [ + 4730, + 990 + ], + "size": [ + 470, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 157 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 24 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "qwen3vl_8b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen3-VL/resolve/main/text_encoders/qwen3vl_8b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen3vl_8b_fp8_scaled.safetensors", + "ideogram4", + "default" + ] + }, + { + "id": 188, + "type": "PrimitiveInt", + "pos": [ + 4240, + 1610 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 153 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 32 + ] + } + ], + "title": "Int (Width)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + "fixed" + ] + }, + { + "id": 189, + "type": "PrimitiveInt", + "pos": [ + 4250, + 1800 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 154 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 35 + ] + } + ], + "title": "Int (Height)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + "fixed" + ] + }, + { + "id": 190, + "type": "ComfyMathExpression", + "pos": [ + 5340, + 1400 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 32 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 33, + 34 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "max(((a + 15) // 16) * 16, 256)" + ] + }, + { + "id": 191, + "type": "ComfyMathExpression", + "pos": [ + 5350, + 1470 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 35 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 36, + 37 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "max(((a + 15) // 16) * 16, 256)" + ] + }, + { + "id": 192, + "type": "ComfyNumberConvert", + "pos": [ + 5920, + 1870 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "label": "value", + "localized_name": "value", + "name": "value", + "type": "INT,FLOAT,STRING,BOOLEAN", + "link": 195 + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 208 + ] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyNumberConvert", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 193, + "type": "JsonExtractString", + "pos": [ + 5450, + 1870 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": 196 + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 195 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "mu" + ] + }, + { + "id": 194, + "type": "ComfyNumberConvert", + "pos": [ + 5930, + 2110 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "label": "value", + "localized_name": "value", + "name": "value", + "type": "INT,FLOAT,STRING,BOOLEAN", + "link": 197 + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 209 + ] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyNumberConvert", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 195, + "type": "JsonExtractString", + "pos": [ + 5010, + 1630 + ], + "size": [ + 410, + 470 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": null + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": 218 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 199 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "{\n \"Quality\": {\n \"num_steps\": 48,\n \"mu\": 0.0,\n \"std\": 1.5,\n \"preset_id\": \"V4_QUALITY_48\"\n },\n \"Default\": {\n \"num_steps\": 20,\n \"mu\": 0.0,\n \"std\": 1.75,\n \"preset_id\": \"V4_DEFAULT_20\"\n },\n \"Turbo\": {\n \"num_steps\": 12,\n \"mu\": 0.5,\n \"std\": 1.75,\n \"preset_id\": \"V4_TURBO_12\"\n }\n}", + "Default" + ] + }, + { + "id": 196, + "type": "StringReplace", + "pos": [ + 5050, + 2150 + ], + "size": [ + 230, + 40 + ], + "flags": { + "collapsed": true + }, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": 199 + }, + { + "localized_name": "find", + "name": "find", + "type": "STRING", + "widget": { + "name": "find" + }, + "link": null + }, + { + "localized_name": "replace", + "name": "replace", + "type": "STRING", + "widget": { + "name": "replace" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 196, + 200, + 201 + ] + } + ], + "properties": { + "Node name for S&R": "StringReplace", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "'", + "\"" + ] + }, + { + "id": 197, + "type": "JsonExtractString", + "pos": [ + 5460, + 1610 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": 200 + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 202 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "num_steps" + ] + }, + { + "id": 198, + "type": "JsonExtractString", + "pos": [ + 5450, + 2110 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": 201 + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 197 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "std" + ] + }, + { + "id": 199, + "type": "ComfyNumberConvert", + "pos": [ + 5920, + 1620 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "value", + "localized_name": "value", + "name": "value", + "type": "INT,FLOAT,STRING,BOOLEAN", + "link": 202 + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 207 + ] + } + ], + "properties": { + "Node name for S&R": "ComfyNumberConvert", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 200, + "type": "UNETLoader", + "pos": [ + 4730, + 740 + ], + "size": [ + 470, + 170 + ], + "flags": {}, + "order": 23, + "mode": 0, + "showAdvanced": true, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 216 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 211 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "ideogram4_unconditional_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Ideogram-4/resolve/main/diffusion_models/ideogram4_unconditional_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "ideogram4_unconditional_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 201, + "type": "DualModelGuider", + "pos": [ + 5790, + 870 + ], + "size": [ + 270, + 180 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 223 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 213 + }, + { + "localized_name": "model_negative", + "name": "model_negative", + "shape": 7, + "type": "MODEL", + "link": 211 + }, + { + "localized_name": "negative", + "name": "negative", + "shape": 7, + "type": "CONDITIONING", + "link": 214 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "GUIDER", + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 215 + ] + } + ], + "properties": { + "Node name for S&R": "DualModelGuider", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 7 + ] + }, + { + "id": 202, + "type": "CustomCombo", + "pos": [ + 4720, + 1630 + ], + "size": [ + 270, + 280 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "choice", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": 219 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 218 + ] + }, + { + "localized_name": "INDEX", + "name": "INDEX", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "CustomCombo", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Default", + 1, + "Quality", + "Default", + "Turbo", + "" + ] + }, + { + "id": 203, + "type": "CFGOverride", + "pos": [ + 5790, + 650 + ], + "size": [ + 260, + 170 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 222 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + }, + { + "localized_name": "start_percent", + "name": "start_percent", + "type": "FLOAT", + "widget": { + "name": "start_percent" + }, + "link": null + }, + { + "localized_name": "end_percent", + "name": "end_percent", + "type": "FLOAT", + "widget": { + "name": "end_percent" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 223 + ] + } + ], + "properties": { + "Node name for S&R": "CFGOverride", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3, + 0.7, + 1 + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 4700, + 420, + 530, + 1100 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Latent Size", + "bounding": [ + 5260, + 1080, + 450, + 440 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Sampling", + "bounding": [ + 5740, + 420, + 780, + 1100 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 5260, + 420, + 450, + 640 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Image Size", + "bounding": [ + 4130, + 1540, + 530, + 420 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 9, + "title": "Preset", + "bounding": [ + 4700, + 1540, + 1820, + 780 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 8, + "origin_id": 186, + "origin_slot": 0, + "target_id": 178, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 33, + "origin_id": 190, + "origin_slot": 1, + "target_id": 179, + "target_slot": 0, + "type": "INT" + }, + { + "id": 36, + "origin_id": 191, + "origin_slot": 1, + "target_id": 179, + "target_slot": 1, + "type": "INT" + }, + { + "id": 11, + "origin_id": 184, + "origin_slot": 0, + "target_id": 180, + "target_slot": 0, + "type": "NOISE" + }, + { + "id": 13, + "origin_id": 182, + "origin_slot": 0, + "target_id": 180, + "target_slot": 2, + "type": "SAMPLER" + }, + { + "id": 14, + "origin_id": 183, + "origin_slot": 0, + "target_id": 180, + "target_slot": 3, + "type": "SIGMAS" + }, + { + "id": 15, + "origin_id": 179, + "origin_slot": 0, + "target_id": 180, + "target_slot": 4, + "type": "LATENT" + }, + { + "id": 16, + "origin_id": 180, + "origin_slot": 0, + "target_id": 181, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 17, + "origin_id": 177, + "origin_slot": 0, + "target_id": 181, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 34, + "origin_id": 190, + "origin_slot": 1, + "target_id": 183, + "target_slot": 1, + "type": "INT" + }, + { + "id": 37, + "origin_id": 191, + "origin_slot": 1, + "target_id": 183, + "target_slot": 2, + "type": "INT" + }, + { + "id": 24, + "origin_id": 187, + "origin_slot": 0, + "target_id": 186, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 32, + "origin_id": 188, + "origin_slot": 0, + "target_id": 190, + "target_slot": 0, + "type": "INT" + }, + { + "id": 35, + "origin_id": 189, + "origin_slot": 0, + "target_id": 191, + "target_slot": 0, + "type": "INT" + }, + { + "id": 25, + "origin_id": 181, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 152, + "origin_id": -10, + "origin_slot": 0, + "target_id": 186, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 153, + "origin_id": -10, + "origin_slot": 1, + "target_id": 188, + "target_slot": 0, + "type": "INT" + }, + { + "id": 154, + "origin_id": -10, + "origin_slot": 2, + "target_id": 189, + "target_slot": 0, + "type": "INT" + }, + { + "id": 155, + "origin_id": -10, + "origin_slot": 3, + "target_id": 184, + "target_slot": 0, + "type": "INT" + }, + { + "id": 156, + "origin_id": -10, + "origin_slot": 4, + "target_id": 185, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 157, + "origin_id": -10, + "origin_slot": 5, + "target_id": 187, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 158, + "origin_id": -10, + "origin_slot": 6, + "target_id": 177, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 195, + "origin_id": 193, + "origin_slot": 0, + "target_id": 192, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 196, + "origin_id": 196, + "origin_slot": 0, + "target_id": 193, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 197, + "origin_id": 198, + "origin_slot": 0, + "target_id": 194, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 199, + "origin_id": 195, + "origin_slot": 0, + "target_id": 196, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 200, + "origin_id": 196, + "origin_slot": 0, + "target_id": 197, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 201, + "origin_id": 196, + "origin_slot": 0, + "target_id": 198, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 202, + "origin_id": 197, + "origin_slot": 0, + "target_id": 199, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 207, + "origin_id": 199, + "origin_slot": 1, + "target_id": 183, + "target_slot": 0, + "type": "INT" + }, + { + "id": 208, + "origin_id": 192, + "origin_slot": 0, + "target_id": 183, + "target_slot": 3, + "type": "FLOAT" + }, + { + "id": 209, + "origin_id": 194, + "origin_slot": 0, + "target_id": 183, + "target_slot": 4, + "type": "FLOAT" + }, + { + "id": 211, + "origin_id": 200, + "origin_slot": 0, + "target_id": 201, + "target_slot": 2, + "type": "MODEL" + }, + { + "id": 213, + "origin_id": 186, + "origin_slot": 0, + "target_id": 201, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 214, + "origin_id": 178, + "origin_slot": 0, + "target_id": 201, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 215, + "origin_id": 201, + "origin_slot": 0, + "target_id": 180, + "target_slot": 1, + "type": "GUIDER" + }, + { + "id": 216, + "origin_id": -10, + "origin_slot": 7, + "target_id": 200, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 218, + "origin_id": 202, + "origin_slot": 0, + "target_id": 195, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 219, + "origin_id": -10, + "origin_slot": 8, + "target_id": 202, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 222, + "origin_id": 185, + "origin_slot": 0, + "target_id": 203, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 223, + "origin_id": 203, + "origin_slot": 0, + "target_id": 201, + "target_slot": 0, + "type": "MODEL" + } + ], + "extra": {}, + "category": "Image generation and editing/Text to image", + "description": "This subgraph generates images using Ideogram v4, accepting plain text or structured JSON prompts for precise layout and style control. It suits detailed illustrations, concept art, or marketing visuals needing predictable composition and color palettes. The model uses flow-matching with asymmetric guidance, so no negative prompt is needed, but JSON prompts yield the best results." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph generates images using Ideogram v4, accepting plain text or structured JSON prompts for precise layout and style control. It suits detailed illustrations, concept art, or marketing visuals needing predictable composition and color palettes. The model uses flow-matching with asymmetric guidance, so no negative prompt is needed, but JSON prompts yield the best results." + } +} \ No newline at end of file diff --git a/blueprints/Video Depth Estimation (Depth Anything 3).json b/blueprints/Video Depth Estimation (Depth Anything 3).json new file mode 100644 index 000000000..88084a393 --- /dev/null +++ b/blueprints/Video Depth Estimation (Depth Anything 3).json @@ -0,0 +1,825 @@ +{ + "revision": 0, + "last_node_id": 97, + "last_link_id": 0, + "nodes": [ + { + "id": 97, + "type": "253ec5ca-8333-4ddf-a036-9fc0923651b9", + "pos": [ + 410, + 500 + ], + "size": [ + 400, + 400 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "name": "start_time", + "type": "FLOAT", + "widget": { + "name": "start_time" + }, + "link": null + }, + { + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": null + }, + { + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": null + }, + { + "label": "output_type", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": null + }, + { + "label": "normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": null + }, + { + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": null + }, + { + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "name": "fps", + "type": "FLOAT", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "96", + "start_time" + ], + [ + "96", + "duration" + ], + [ + "93", + "resolution" + ], + [ + "93", + "resize_method" + ], + [ + "92", + "output" + ], + [ + "92", + "output.normalization" + ], + [ + "92", + "output.apply_sky_clip" + ], + [ + "94", + "model_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Video Depth Estimation (Depth Anything 3)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "253ec5ca-8333-4ddf-a036-9fc0923651b9", + "version": 1, + "state": { + "lastGroupId": 4, + "lastNodeId": 97, + "lastLinkId": 129, + "lastRerouteId": 0 + }, + "revision": 2, + "config": {}, + "name": "Video Depth Estimation (Depth Anything 3)", + "inputNode": { + "id": -10, + "bounding": [ + -230, + 130, + 167.912109375, + 228 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1520, + 140, + 128, + 108 + ] + }, + "inputs": [ + { + "id": "698c28c6-cf92-4039-8b39-f3062868ea7c", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 119 + ], + "pos": [ + -86.087890625, + 154 + ] + }, + { + "id": "97a1f63e-1585-4a40-9dec-e2700120d84a", + "name": "start_time", + "type": "FLOAT", + "linkIds": [ + 121 + ], + "pos": [ + -86.087890625, + 174 + ] + }, + { + "id": "4dbbd3b3-c5ee-4a56-a0d3-3268d3b2fd64", + "name": "duration", + "type": "FLOAT", + "linkIds": [ + 122 + ], + "pos": [ + -86.087890625, + 194 + ] + }, + { + "id": "16f55101-f99d-4c0c-bebf-c3b31c54f13e", + "name": "resolution", + "type": "INT", + "linkIds": [ + 124 + ], + "pos": [ + -86.087890625, + 214 + ] + }, + { + "id": "d9cd7693-4bb3-4ed7-9a75-276b997abcd9", + "name": "resize_method", + "type": "COMBO", + "linkIds": [ + 125 + ], + "pos": [ + -86.087890625, + 234 + ] + }, + { + "id": "a6e90532-323b-462e-ba9c-1672384d5b31", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "linkIds": [ + 126 + ], + "label": "output_type", + "pos": [ + -86.087890625, + 254 + ] + }, + { + "id": "69e6aeef-437d-4fde-b2fc-d5ab9369238d", + "name": "output.normalization", + "type": "COMBO", + "linkIds": [ + 127 + ], + "label": "normalization", + "pos": [ + -86.087890625, + 274 + ] + }, + { + "id": "73206f72-f89a-4698-885e-5d9277df2998", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "linkIds": [ + 128 + ], + "pos": [ + -86.087890625, + 294 + ] + }, + { + "id": "dddbc7fc-9431-448a-9ed3-9aa62404288b", + "name": "model_name", + "type": "COMBO", + "linkIds": [ + 129 + ], + "pos": [ + -86.087890625, + 314 + ] + } + ], + "outputs": [ + { + "id": "478ab537-63bc-4d74-a9f0-c975f550880f", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 7 + ], + "localized_name": "IMAGE", + "pos": [ + 1544, + 164 + ] + }, + { + "id": "cdaf037e-79bc-4a94-b06c-0fd32e76f615", + "name": "audio", + "type": "AUDIO", + "linkIds": [ + 112 + ], + "pos": [ + 1544, + 184 + ] + }, + { + "id": "4c0e5484-d193-49c7-b107-92619628880a", + "name": "fps", + "type": "FLOAT", + "linkIds": [ + 113 + ], + "pos": [ + 1544, + 204 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 92, + "type": "DA3Render", + "pos": [ + 740, + 230 + ], + "size": [ + 380, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "link": 12 + }, + { + "localized_name": "output", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": 126 + }, + { + "localized_name": "output.normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": 127 + }, + { + "localized_name": "output.apply_sky_clip", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": 128 + }, + { + "name": "geometry", + "type": "DA3_GEOMETRY", + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Render", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + "depth", + "v2_style", + false + ] + }, + { + "id": 93, + "type": "DA3Inference", + "pos": [ + 740, + -30 + ], + "size": [ + 390, + 130 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_model", + "name": "da3_model", + "type": "DA3_MODEL", + "link": 107 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 111 + }, + { + "localized_name": "resolution", + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": 124 + }, + { + "localized_name": "resize_method", + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": 125 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "slot_index": 0, + "links": [ + 12 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Inference", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + 504, + "lower_bound_resize", + "mono" + ] + }, + { + "id": 94, + "type": "LoadDA3Model", + "pos": [ + 50, + 410 + ], + "size": [ + 400, + 140 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "model_name", + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": 129 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "DA3_MODEL", + "name": "DA3_MODEL", + "type": "DA3_MODEL", + "links": [ + 107 + ] + } + ], + "properties": { + "Node name for S&R": "LoadDA3Model", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "depth_anything_3_mono_large.safetensors", + "url": "https://huggingface.co/Comfy-Org/Depth-Anything-3/resolve/main/geometry_estimation/depth_anything_3_mono_large.safetensors", + "directory": "geometry_estimation" + } + ] + }, + "widgets_values": [ + "depth_anything_3_mono_large.safetensors", + "default" + ] + }, + { + "id": 95, + "type": "GetVideoComponents", + "pos": [ + 70, + -140 + ], + "size": [ + 260, + 120 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 120 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 111 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [ + 112 + ] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 113 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 96, + "type": "Video Slice", + "pos": [ + 70, + -360 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 119 + }, + { + "localized_name": "start_time", + "name": "start_time", + "type": "FLOAT", + "widget": { + "name": "start_time" + }, + "link": 121 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": 122 + }, + { + "localized_name": "strict_duration", + "name": "strict_duration", + "type": "BOOLEAN", + "widget": { + "name": "strict_duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 120 + ] + } + ], + "properties": { + "Node name for S&R": "Video Slice", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0, + 5, + false + ] + } + ], + "groups": [], + "links": [ + { + "id": 12, + "origin_id": 93, + "origin_slot": 0, + "target_id": 92, + "target_slot": 0, + "type": "DA3_GEOMETRY" + }, + { + "id": 7, + "origin_id": 92, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 107, + "origin_id": 94, + "origin_slot": 0, + "target_id": 93, + "target_slot": 0, + "type": "DA3_MODEL" + }, + { + "id": 111, + "origin_id": 95, + "origin_slot": 0, + "target_id": 93, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 112, + "origin_id": 95, + "origin_slot": 1, + "target_id": -20, + "target_slot": 1, + "type": "AUDIO" + }, + { + "id": 113, + "origin_id": 95, + "origin_slot": 2, + "target_id": -20, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 119, + "origin_id": -10, + "origin_slot": 0, + "target_id": 96, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 120, + "origin_id": 96, + "origin_slot": 0, + "target_id": 95, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 121, + "origin_id": -10, + "origin_slot": 1, + "target_id": 96, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 122, + "origin_id": -10, + "origin_slot": 2, + "target_id": 96, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 124, + "origin_id": -10, + "origin_slot": 3, + "target_id": 93, + "target_slot": 2, + "type": "INT" + }, + { + "id": 125, + "origin_id": -10, + "origin_slot": 4, + "target_id": 93, + "target_slot": 3, + "type": "COMBO" + }, + { + "id": 126, + "origin_id": -10, + "origin_slot": 5, + "target_id": 92, + "target_slot": 1, + "type": "COMFY_DYNAMICCOMBO_V3" + }, + { + "id": 127, + "origin_id": -10, + "origin_slot": 6, + "target_id": 92, + "target_slot": 2, + "type": "COMBO" + }, + { + "id": 128, + "origin_id": -10, + "origin_slot": 7, + "target_id": 92, + "target_slot": 3, + "type": "BOOLEAN" + }, + { + "id": 129, + "origin_id": -10, + "origin_slot": 8, + "target_id": 94, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Conditioning & Preprocessors/Depth", + "description": "This subgraph processes a video input through Depth Anything 3 to produce temporally consistent depth maps for each frame, outputting a depth video. It is ideal for video content requiring spatial geometry estimation, such as 3D reconstruction, SLAM, or novel view synthesis from moving cameras. The model uses a plain transformer backbone trained with a depth-ray representation, supporting any number of views without requiring known camera poses." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph processes a video input through Depth Anything 3 to produce temporally consistent depth maps for each frame, outputting a depth video. It is ideal for video content requiring spatial geometry estimation, such as 3D reconstruction, SLAM, or novel view synthesis from moving cameras. The model uses a plain transformer backbone trained with a depth-ray representation, supporting any number of views without requiring known camera poses." + } +} \ No newline at end of file diff --git a/blueprints/Video Edit (Bernini-R).json b/blueprints/Video Edit (Bernini-R).json new file mode 100644 index 000000000..2f09db5fa --- /dev/null +++ b/blueprints/Video Edit (Bernini-R).json @@ -0,0 +1,3732 @@ +{ + "revision": 0, + "last_node_id": 376, + "last_link_id": 0, + "nodes": [ + { + "id": 376, + "type": "70d8911e-8530-4a3d-9889-b39e8fbd131b", + "pos": [ + 4090, + 4890 + ], + "size": [ + 480, + 740 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "source_video", + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "label": "reference_images", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "value", + "type": "STRING", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "task_type", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "label": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + }, + { + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": null + }, + { + "label": "turbo_mode", + "name": "value_1", + "type": "BOOLEAN", + "widget": { + "name": "value_1" + }, + "link": null + }, + { + "label": "high_noise_model", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "low_noise_model", + "name": "unet_name_1", + "type": "COMBO", + "widget": { + "name": "unet_name_1" + }, + "link": null + }, + { + "label": "distill_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "371", + "value" + ], + [ + "356", + "choice" + ], + [ + "352", + "width" + ], + [ + "352", + "height" + ], + [ + "349", + "noise_seed" + ], + [ + "352", + "length" + ], + [ + "352", + "ref_max_size" + ], + [ + "368", + "value" + ], + [ + "344", + "unet_name" + ], + [ + "346", + "unet_name" + ], + [ + "345", + "lora_name" + ], + [ + "338", + "clip_name" + ], + [ + "339", + "vae_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Video Edit (Bernini-R)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "70d8911e-8530-4a3d-9889-b39e8fbd131b", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 376, + "lastLinkId": 496, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Video Edit (Bernini-R)", + "inputNode": { + "id": -10, + "bounding": [ + 5240, + 5360, + 149.689453125, + 368 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 10004.984375, + 5000.9921875, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "15c3bfa3-6844-473a-a927-a50284131356", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 457 + ], + "localized_name": "video", + "label": "source_video", + "pos": [ + 5365.689453125, + 5384 + ] + }, + { + "id": "c02ac440-58ae-4415-bb24-dabb61b91f69", + "name": "reference_video", + "type": "IMAGE", + "linkIds": [ + 465 + ], + "pos": [ + 5365.689453125, + 5404 + ] + }, + { + "id": "c46d38f5-b0e8-4620-bd0e-4e86db7bdb1b", + "name": "reference_images.reference_image_0", + "type": "IMAGE", + "linkIds": [ + 466 + ], + "label": "reference_images", + "pos": [ + 5365.689453125, + 5424 + ] + }, + { + "id": "565fc711-6052-4c07-b638-403e01fcf7f8", + "name": "value", + "type": "STRING", + "linkIds": [ + 467 + ], + "pos": [ + 5365.689453125, + 5444 + ] + }, + { + "id": "bfcd6148-0bae-43b1-9440-a133fbc2663c", + "name": "choice", + "type": "COMBO", + "linkIds": [ + 468 + ], + "label": "task_type", + "pos": [ + 5365.689453125, + 5464 + ] + }, + { + "id": "28fedb03-828c-49d9-91ec-d3669a10c3b9", + "name": "width", + "type": "INT", + "linkIds": [ + 469 + ], + "pos": [ + 5365.689453125, + 5484 + ] + }, + { + "id": "69994b05-577e-486f-bd10-3360261d7bb8", + "name": "height", + "type": "INT", + "linkIds": [ + 470 + ], + "pos": [ + 5365.689453125, + 5504 + ] + }, + { + "id": "db300c09-5b92-41ad-990b-9dd8dad35f86", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 480 + ], + "label": "noise_seed", + "pos": [ + 5365.689453125, + 5524 + ] + }, + { + "id": "db664a31-39e5-4d6a-a5f1-3abac65b35d1", + "name": "length", + "type": "INT", + "linkIds": [ + 481 + ], + "pos": [ + 5365.689453125, + 5544 + ] + }, + { + "id": "26aa0c9e-9daa-4302-ab5c-5ac9141b9e20", + "name": "ref_max_size", + "type": "INT", + "linkIds": [ + 482 + ], + "pos": [ + 5365.689453125, + 5564 + ] + }, + { + "id": "ee527a92-8cc9-4b16-9858-9daab9ef2c45", + "name": "value_1", + "type": "BOOLEAN", + "linkIds": [ + 488 + ], + "label": "turbo_mode", + "pos": [ + 5365.689453125, + 5584 + ] + }, + { + "id": "8a6c9f3d-e24e-4d40-9c03-864bd4458376", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 489 + ], + "label": "high_noise_model", + "pos": [ + 5365.689453125, + 5604 + ] + }, + { + "id": "f35afd1c-c183-4d18-8672-314527728e9b", + "name": "unet_name_1", + "type": "COMBO", + "linkIds": [ + 490 + ], + "label": "low_noise_model", + "pos": [ + 5365.689453125, + 5624 + ] + }, + { + "id": "fff89f1d-615c-436e-b28a-3b7f915d0b05", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 491, + 492 + ], + "label": "distill_lora", + "pos": [ + 5365.689453125, + 5644 + ] + }, + { + "id": "d76ff30d-c865-49b4-bccb-fb6e0a9b4f34", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 493 + ], + "pos": [ + 5365.689453125, + 5664 + ] + }, + { + "id": "0850a515-4051-4de3-9343-7db929548ada", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 494 + ], + "pos": [ + 5365.689453125, + 5684 + ] + } + ], + "outputs": [ + { + "id": "7d994238-c919-43c6-9d97-340c9e383743", + "name": "VIDEO", + "type": "VIDEO", + "linkIds": [ + 458 + ], + "localized_name": "VIDEO", + "pos": [ + 10028.984375, + 5024.9921875 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 338, + "type": "CLIPLoader", + "pos": [ + 6170, + 5760 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 493 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 409, + 438 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors?download=true", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 339, + "type": "VAELoader", + "pos": [ + 6170, + 5990 + ], + "size": [ + 670, + 110 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 494 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 414, + 436 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors?download=true", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 340, + "type": "LoraLoaderModelOnly", + "pos": [ + 6180, + 5550 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 408 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 492 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 441 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 1.5 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 341, + "type": "CLIPTextEncode", + "pos": [ + 6950, + 5400 + ], + "size": [ + 700, + 240 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 409 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 435 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 342, + "type": "SplitSigmas", + "pos": [ + 8520, + 5180 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 410 + }, + { + "localized_name": "step", + "name": "step", + "type": "INT", + "widget": { + "name": "step" + }, + "link": 411 + } + ], + "outputs": [ + { + "localized_name": "high_sigmas", + "name": "high_sigmas", + "type": "SIGMAS", + "links": [ + 422 + ] + }, + { + "localized_name": "low_sigmas", + "name": "low_sigmas", + "type": "SIGMAS", + "links": [ + 431 + ] + } + ], + "properties": { + "Node name for S&R": "SplitSigmas", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 343, + "type": "KSamplerSelect", + "pos": [ + 8520, + 5370 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 421, + 430 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "res_multistep" + ] + }, + { + "id": 344, + "type": "UNETLoader", + "pos": [ + 6170, + 4930 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 489 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 412, + 454 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 345, + "type": "LoraLoaderModelOnly", + "pos": [ + 6170, + 5120 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 412 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 491 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 455 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 3 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 346, + "type": "UNETLoader", + "pos": [ + 6170, + 5350 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 490 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 408, + 425, + 440 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 347, + "type": "VAEDecode", + "pos": [ + 9690, + 4950 + ], + "size": [ + 250, + 100 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 413 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 414 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 415 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 348, + "type": "CreateVideo", + "pos": [ + 9690, + 5120 + ], + "size": [ + 260, + 160 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 415 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 416 + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 417 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "INT", + "widget": { + "name": "bit_depth" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 458 + ] + } + ], + "properties": { + "Node name for S&R": "CreateVideo", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 24, + 8 + ] + }, + { + "id": 349, + "type": "SamplerCustom", + "pos": [ + 8860, + 4960 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 18, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 418 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 419 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 420 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 421 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 422 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 423 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 480 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 424 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 432 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true, + 852303761886160, + "randomize", + 1 + ] + }, + { + "id": 350, + "type": "BasicScheduler", + "pos": [ + 8520, + 4960 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 425 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 426 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 410 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 351, + "type": "SamplerCustom", + "pos": [ + 9190, + 4950 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 20, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 427 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 428 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 429 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 430 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 431 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 432 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 433 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 413 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + false, + 0, + "fixed", + 1 + ] + }, + { + "id": 352, + "type": "BerniniConditioning", + "pos": [ + 7160, + 5720 + ], + "size": [ + 310, + 380 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 434 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 435 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 436 + }, + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "IMAGE", + "link": 437 + }, + { + "localized_name": "reference_video", + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": 465 + }, + { + "label": "reference_image_0", + "localized_name": "reference_images.reference_image_0", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": 466 + }, + { + "label": "reference_image_1", + "localized_name": "reference_images.reference_image_1", + "name": "reference_images.reference_image_1", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 469 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 470 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 481 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "ref_max_size", + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": 482 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 419, + 428 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 420, + 429 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 423 + ] + } + ], + "properties": { + "Node name for S&R": "BerniniConditioning", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 480, + 832, + 81, + 1, + 848 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 353, + "type": "GetVideoComponents", + "pos": [ + 6170, + 6220 + ], + "size": [ + 230, + 150 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 457 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 437 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [ + 416 + ] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 417 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 354, + "type": "CLIPTextEncode", + "pos": [ + 6950, + 4940 + ], + "size": [ + 710, + 390 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 438 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 439 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 434 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 355, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5250 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 440 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 441 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 442 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 427 + ] + } + ], + "title": "Switch (Low Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 356, + "type": "CustomCombo", + "pos": [ + 6170, + 3800 + ], + "size": [ + 460, + 600 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "choice", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": 468 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [] + }, + { + "localized_name": "INDEX", + "name": "INDEX", + "type": "INT", + "links": [ + 443 + ] + } + ], + "properties": { + "Node name for S&R": "CustomCombo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "Video Editing (Content Propagation)", + 7, + "Default", + "Text to Image", + "Text to Video", + "Image Editing", + "Subject to Image", + "Image to Video", + "Video Editing", + "Video Editing (Content Propagation)", + "Video Editing with Reference", + "Ads / Content Insertion", + "Video Editing (Action / Position)", + "Video Editing (Style / Motion)", + "" + ] + }, + { + "id": 357, + "type": "a98d3dcb-12b1-467c-94b8-723a89533c30", + "pos": [ + 6680, + 3800 + ], + "size": [ + 390, + 440 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "text_per_line", + "name": "text_per_line", + "type": "STRING", + "widget": { + "name": "text_per_line" + }, + "link": null + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "widget": { + "name": "index" + }, + "link": 443 + } + ], + "outputs": [ + { + "localized_name": "selected_line", + "name": "selected_line", + "type": "STRING", + "links": [ + 444 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "373", + "string" + ], + [ + "374", + "value" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 358, + "type": "StringConcatenate", + "pos": [ + 6680, + 4500 + ], + "size": [ + 400, + 250 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "string_a", + "name": "string_a", + "type": "STRING", + "widget": { + "name": "string_a" + }, + "link": 444 + }, + { + "localized_name": "string_b", + "name": "string_b", + "type": "STRING", + "widget": { + "name": "string_b" + }, + "link": 459 + }, + { + "localized_name": "delimiter", + "name": "delimiter", + "type": "STRING", + "widget": { + "name": "delimiter" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 439 + ] + } + ], + "properties": { + "Node name for S&R": "StringConcatenate", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "", + "" + ] + }, + { + "id": 359, + "type": "PrimitiveInt", + "pos": [ + 7740, + 4970 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 445 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 360, + "type": "PrimitiveInt", + "pos": [ + 7750, + 5510 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 446 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 361, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5440 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 445 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 446 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 447 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 426 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 362, + "type": "PrimitiveInt", + "pos": [ + 7740, + 5130 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 448 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 20, + "fixed" + ] + }, + { + "id": 363, + "type": "PrimitiveInt", + "pos": [ + 7750, + 5680 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 449 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 3, + "fixed" + ] + }, + { + "id": 364, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5640 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 448 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 449 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 450 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 411 + ] + } + ], + "title": "Switch (Low Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 365, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5830 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 451 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 452 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 453 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 424, + 433 + ] + } + ], + "title": "Switch (CFG)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 366, + "type": "PrimitiveFloat", + "pos": [ + 7750, + 5840 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 452 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 367, + "type": "PrimitiveFloat", + "pos": [ + 7740, + 5290 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 451 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 368, + "type": "PrimitiveBoolean", + "pos": [ + 7750, + 6020 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 488 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 442, + 447, + 450, + 453, + 456 + ] + } + ], + "title": "Boolean (Enable Turbo LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 369, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5060 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 454 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 455 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 456 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 418 + ] + } + ], + "title": "Switch (High Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 370, + "type": "MarkdownNote", + "pos": [ + 7730, + 4680 + ], + "size": [ + 340, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can find the original settings here: https://github.com/bytedance/Bernini/blob/main/gradio_demo.py" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 371, + "type": "PrimitiveStringMultiline", + "pos": [ + 6160, + 4510 + ], + "size": [ + 470, + 230 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "STRING", + "widget": { + "name": "value" + }, + "link": 467 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 459 + ] + } + ], + "properties": { + "Node name for S&R": "PrimitiveStringMultiline", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "Replace the gray studio backdrop with a daytime urban street: brick buildings, shop windows, sidewalk, and soft overcast light. Keep the model's outfit, accessories, body pose, motion, and full-body framing unchanged. Only the environment behind the subject should change." + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Prompt & Conditioning", + "bounding": [ + 6930, + 4860, + 760, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Models", + "bounding": [ + 6150, + 4860, + 750, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Sampling", + "bounding": [ + 8490, + 4860, + 1160, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Prompt Construction", + "bounding": [ + 6150, + 3690, + 960, + 1120 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "System Prompt (Based on task type)", + "bounding": [ + 6160, + 3730, + 920, + 680 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Switch Settings", + "bounding": [ + 7720, + 4860, + 740, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Distill LoRA settings", + "bounding": [ + 7730, + 5440, + 310, + 530 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "Original Settings", + "bounding": [ + 7730, + 4900, + 300, + 504 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 408, + "origin_id": 346, + "origin_slot": 0, + "target_id": 340, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 409, + "origin_id": 338, + "origin_slot": 0, + "target_id": 341, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 410, + "origin_id": 350, + "origin_slot": 0, + "target_id": 342, + "target_slot": 0, + "type": "SIGMAS" + }, + { + "id": 411, + "origin_id": 364, + "origin_slot": 0, + "target_id": 342, + "target_slot": 1, + "type": "INT" + }, + { + "id": 412, + "origin_id": 344, + "origin_slot": 0, + "target_id": 345, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 413, + "origin_id": 351, + "origin_slot": 0, + "target_id": 347, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 414, + "origin_id": 339, + "origin_slot": 0, + "target_id": 347, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 415, + "origin_id": 347, + "origin_slot": 0, + "target_id": 348, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 416, + "origin_id": 353, + "origin_slot": 1, + "target_id": 348, + "target_slot": 1, + "type": "AUDIO" + }, + { + "id": 417, + "origin_id": 353, + "origin_slot": 2, + "target_id": 348, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 418, + "origin_id": 369, + "origin_slot": 0, + "target_id": 349, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 419, + "origin_id": 352, + "origin_slot": 0, + "target_id": 349, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 420, + "origin_id": 352, + "origin_slot": 1, + "target_id": 349, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 421, + "origin_id": 343, + "origin_slot": 0, + "target_id": 349, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 422, + "origin_id": 342, + "origin_slot": 0, + "target_id": 349, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 423, + "origin_id": 352, + "origin_slot": 2, + "target_id": 349, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 424, + "origin_id": 365, + "origin_slot": 0, + "target_id": 349, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 425, + "origin_id": 346, + "origin_slot": 0, + "target_id": 350, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 426, + "origin_id": 361, + "origin_slot": 0, + "target_id": 350, + "target_slot": 2, + "type": "INT" + }, + { + "id": 427, + "origin_id": 355, + "origin_slot": 0, + "target_id": 351, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 428, + "origin_id": 352, + "origin_slot": 0, + "target_id": 351, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 429, + "origin_id": 352, + "origin_slot": 1, + "target_id": 351, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 430, + "origin_id": 343, + "origin_slot": 0, + "target_id": 351, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 431, + "origin_id": 342, + "origin_slot": 1, + "target_id": 351, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 432, + "origin_id": 349, + "origin_slot": 0, + "target_id": 351, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 433, + "origin_id": 365, + "origin_slot": 0, + "target_id": 351, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 434, + "origin_id": 354, + "origin_slot": 0, + "target_id": 352, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 435, + "origin_id": 341, + "origin_slot": 0, + "target_id": 352, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 436, + "origin_id": 339, + "origin_slot": 0, + "target_id": 352, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 437, + "origin_id": 353, + "origin_slot": 0, + "target_id": 352, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 438, + "origin_id": 338, + "origin_slot": 0, + "target_id": 354, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 439, + "origin_id": 358, + "origin_slot": 0, + "target_id": 354, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 440, + "origin_id": 346, + "origin_slot": 0, + "target_id": 355, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 441, + "origin_id": 340, + "origin_slot": 0, + "target_id": 355, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 442, + "origin_id": 368, + "origin_slot": 0, + "target_id": 355, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 443, + "origin_id": 356, + "origin_slot": 1, + "target_id": 357, + "target_slot": 1, + "type": "INT" + }, + { + "id": 444, + "origin_id": 357, + "origin_slot": 0, + "target_id": 358, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 459, + "origin_id": 371, + "origin_slot": 0, + "target_id": 358, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 445, + "origin_id": 359, + "origin_slot": 0, + "target_id": 361, + "target_slot": 0, + "type": "INT" + }, + { + "id": 446, + "origin_id": 360, + "origin_slot": 0, + "target_id": 361, + "target_slot": 1, + "type": "INT" + }, + { + "id": 447, + "origin_id": 368, + "origin_slot": 0, + "target_id": 361, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 448, + "origin_id": 362, + "origin_slot": 0, + "target_id": 364, + "target_slot": 0, + "type": "INT" + }, + { + "id": 449, + "origin_id": 363, + "origin_slot": 0, + "target_id": 364, + "target_slot": 1, + "type": "INT" + }, + { + "id": 450, + "origin_id": 368, + "origin_slot": 0, + "target_id": 364, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 451, + "origin_id": 367, + "origin_slot": 0, + "target_id": 365, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 452, + "origin_id": 366, + "origin_slot": 0, + "target_id": 365, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 453, + "origin_id": 368, + "origin_slot": 0, + "target_id": 365, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 454, + "origin_id": 344, + "origin_slot": 0, + "target_id": 369, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 455, + "origin_id": 345, + "origin_slot": 0, + "target_id": 369, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 456, + "origin_id": 368, + "origin_slot": 0, + "target_id": 369, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 457, + "origin_id": -10, + "origin_slot": 0, + "target_id": 353, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 458, + "origin_id": 348, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 465, + "origin_id": -10, + "origin_slot": 1, + "target_id": 352, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 466, + "origin_id": -10, + "origin_slot": 2, + "target_id": 352, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 467, + "origin_id": -10, + "origin_slot": 3, + "target_id": 371, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 468, + "origin_id": -10, + "origin_slot": 4, + "target_id": 356, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 469, + "origin_id": -10, + "origin_slot": 5, + "target_id": 352, + "target_slot": 7, + "type": "INT" + }, + { + "id": 470, + "origin_id": -10, + "origin_slot": 6, + "target_id": 352, + "target_slot": 8, + "type": "INT" + }, + { + "id": 480, + "origin_id": -10, + "origin_slot": 7, + "target_id": 349, + "target_slot": 7, + "type": "INT" + }, + { + "id": 481, + "origin_id": -10, + "origin_slot": 8, + "target_id": 352, + "target_slot": 9, + "type": "INT" + }, + { + "id": 482, + "origin_id": -10, + "origin_slot": 9, + "target_id": 352, + "target_slot": 11, + "type": "INT" + }, + { + "id": 488, + "origin_id": -10, + "origin_slot": 10, + "target_id": 368, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 489, + "origin_id": -10, + "origin_slot": 11, + "target_id": 344, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 490, + "origin_id": -10, + "origin_slot": 12, + "target_id": 346, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 491, + "origin_id": -10, + "origin_slot": 13, + "target_id": 345, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 492, + "origin_id": -10, + "origin_slot": 13, + "target_id": 340, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 493, + "origin_id": -10, + "origin_slot": 14, + "target_id": 338, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 494, + "origin_id": -10, + "origin_slot": 15, + "target_id": 339, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Video generation and editing/Video Edit", + "description": "This subgraph uses Depth Anything 3 to predict spatially consistent geometry from any number of images or video frames, with or without known camera poses. It outputs depth maps, camera poses, and optionally 3D Gaussian parameters for novel view synthesis." + }, + { + "id": "a98d3dcb-12b1-467c-94b8-723a89533c30", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 376, + "lastLinkId": 496, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Select Per-Line Text by Index", + "description": "Selects one line from multiline text by zero-based index for batch or list-driven prompt workflows.", + "inputNode": { + "id": -10, + "bounding": [ + -990, + 8595, + 128, + 88 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 710, + 8585, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "75417d82-a934-4ac9-b667-d8dcd5a3bfb3", + "name": "text_per_line", + "type": "STRING", + "linkIds": [ + 13 + ], + "localized_name": "text_per_line", + "pos": [ + -886, + 8619 + ] + }, + { + "id": "46e69a73-1804-4ca6-9175-31445bf0be96", + "name": "index", + "type": "INT", + "linkIds": [ + 14 + ], + "localized_name": "index", + "pos": [ + -886, + 8639 + ] + } + ], + "outputs": [ + { + "id": "e34e8ad1-84d2-4bd2-a460-eb7de6067c10", + "name": "selected_line", + "type": "STRING", + "linkIds": [ + 10 + ], + "localized_name": "selected_line", + "pos": [ + 734, + 8609 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 372, + "type": "PreviewAny", + "pos": [ + -500, + 8400 + ], + "size": [ + 230, + 180 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "PreviewAny", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 373, + "type": "RegexExtract", + "pos": [ + -240, + 8740 + ], + "size": [ + 470, + 460 + ], + "flags": {}, + "order": 1, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": 13 + }, + { + "localized_name": "regex_pattern", + "name": "regex_pattern", + "type": "STRING", + "widget": { + "name": "regex_pattern" + }, + "link": 9 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "case_insensitive", + "name": "case_insensitive", + "type": "BOOLEAN", + "widget": { + "name": "case_insensitive" + }, + "link": null + }, + { + "localized_name": "multiline", + "name": "multiline", + "type": "BOOLEAN", + "widget": { + "name": "multiline" + }, + "link": null + }, + { + "localized_name": "dotall", + "name": "dotall", + "type": "BOOLEAN", + "widget": { + "name": "dotall" + }, + "link": null + }, + { + "localized_name": "group_index", + "name": "group_index", + "type": "INT", + "widget": { + "name": "group_index" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "RegexExtract", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "You are a helpful assistant.\nYou are a helpful assistant specialized in text-to-image generation.\nYou are a helpful assistant specialized in text-to-video generation.\nYou are a helpful assistant specialized in image editing.\nYou are a helpful assistant specialized in subject-to-image generation.\nYou are a helpful assistant specialized in image-to-video generation.\nYou are a helpful assistant specialized in video editing.\nYou are a helpful assistant specialized in video editing on content propagation.\nYou are a helpful assistant specialized in video editing with reference.\nYou are a helpful assistant specialized in ads insertion.\nYou are a helpful assistant for editing. You may need to adjust the subject's action or position.\nYou are a helpful assistant for editing. You might need to adjust the video's style, lighting, colors, textures, and the subject's pose or action.", + "", + "First Group", + false, + false, + false, + 1 + ] + }, + { + "id": 374, + "type": "PrimitiveInt", + "pos": [ + -810, + 8400 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 14 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 1 + ] + } + ], + "title": "Int (line index)", + "properties": { + "Node name for S&R": "Int (line index)", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 0, + "fixed" + ] + }, + { + "id": 375, + "type": "StringReplace", + "pos": [ + -240, + 8400 + ], + "size": [ + 400, + 280 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": null + }, + { + "localized_name": "find", + "name": "find", + "type": "STRING", + "widget": { + "name": "find" + }, + "link": null + }, + { + "localized_name": "replace", + "name": "replace", + "type": "STRING", + "widget": { + "name": "replace" + }, + "link": 6 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 9 + ] + } + ], + "properties": { + "Node name for S&R": "StringReplace", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "^(?:[^\\n]*\\n){index}([^\\n]*)(?:\\n|$)", + "index", + "" + ] + } + ], + "groups": [], + "links": [ + { + "id": 1, + "origin_id": 374, + "origin_slot": 0, + "target_id": 372, + "target_slot": 0, + "type": "INT" + }, + { + "id": 9, + "origin_id": 375, + "origin_slot": 0, + "target_id": 373, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 6, + "origin_id": 372, + "origin_slot": 0, + "target_id": 375, + "target_slot": 2, + "type": "STRING" + }, + { + "id": 10, + "origin_id": 373, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 13, + "origin_id": -10, + "origin_slot": 0, + "target_id": 373, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 14, + "origin_id": -10, + "origin_slot": 1, + "target_id": 374, + "target_slot": 0, + "type": "INT" + } + ], + "extra": { + "ue_links": [], + "links_added_by_ue": [] + } + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph uses Depth Anything 3 to predict spatially consistent geometry from any number of images or video frames, with or without known camera poses. It outputs depth maps, camera poses, and optionally 3D Gaussian parameters for novel view synthesis." + } +} \ No newline at end of file diff --git a/comfy/cli_args.py b/comfy/cli_args.py index a4cabcc65..e2e0d97ec 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -92,6 +92,7 @@ parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE" parser.add_argument("--oneapi-device-selector", type=str, default=None, metavar="SELECTOR_STRING", help="Sets the oneAPI device(s) this instance will use.") parser.add_argument("--supports-fp8-compute", action="store_true", help="ComfyUI will act like if the device supports fp8 compute.") parser.add_argument("--enable-triton-backend", action="store_true", help="ComfyUI will enable the use of Triton backend in comfy-kitchen. Is disabled at launch by default.") +parser.add_argument("--disable-triton-backend", action="store_true", help="Force-disable the comfy-kitchen Triton backend, overriding the automatic ROCm/AMD default and --enable-triton-backend.") class LatentPreviewMethod(enum.Enum): NoPreviews = "none" @@ -115,6 +116,7 @@ cache_group.add_argument("--cache-ram", nargs='*', type=float, default=[], metav cache_group.add_argument("--cache-classic", action="store_true", help="Use the old style (aggressive) caching.") cache_group.add_argument("--cache-lru", type=int, default=0, help="Use LRU caching with a maximum of N node results cached. May use more RAM/VRAM.") cache_group.add_argument("--cache-none", action="store_true", help="Reduced RAM/VRAM usage at the expense of executing every node for each run.") +cache_group.add_argument("--high-ram", action="store_true", help="Can improve performance slightly on high RAM or on systems where pagefile use is preferred over model loading.") attn_group = parser.add_mutually_exclusive_group() attn_group.add_argument("--use-split-cross-attention", action="store_true", help="Use the split cross attention optimization. Ignored when xformers is used.") @@ -133,7 +135,7 @@ upcast.add_argument("--dont-upcast-attention", action="store_true", help="Disabl parser.add_argument("--enable-manager", action="store_true", help="Enable the ComfyUI-Manager feature.") manager_group = parser.add_mutually_exclusive_group() manager_group.add_argument("--disable-manager-ui", action="store_true", help="Disables only the ComfyUI-Manager UI and endpoints. Scheduled installations and similar background tasks will still operate.") -manager_group.add_argument("--enable-manager-legacy-ui", action="store_true", help="Enables the legacy UI of ComfyUI-Manager") +manager_group.add_argument("--enable-manager-legacy-ui", action="store_true", help="Enables the legacy UI of ComfyUI-Manager. Implies --enable-manager.") vram_group = parser.add_mutually_exclusive_group() @@ -144,6 +146,7 @@ vram_group.add_argument("--novram", action="store_true", help="When lowvram isn' vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") parser.add_argument("--reserve-vram", type=float, default=None, help="Set the amount of vram in GB you want to reserve for use by your OS/other software. By default some amount is reserved depending on your OS.") +parser.add_argument("--vram-headroom", type=float, default=0, help="Set the amount of vram in GB for DynamicVRAM to maintain as extra headroom above default. ComfyUI will try and keep this much VRAM completely free and unused, even counting VRAM from other apps.") parser.add_argument("--async-offload", nargs='?', const=2, type=int, default=None, metavar="NUM_STREAMS", help="Use async weight offloading. An optional argument controls the amount of offload streams. Default is 2. Enabled by default on Nvidia.") parser.add_argument("--disable-async-offload", action="store_true", help="Disable async weight offloading.") @@ -166,6 +169,8 @@ class PerformanceFeature(enum.Enum): parser.add_argument("--fast", nargs="*", type=PerformanceFeature, help="Enable some untested and potentially quality deteriorating optimizations. This is used to test new features so using it might crash your comfyui. --fast with no arguments enables everything. You can pass a list specific optimizations if you only want to enable specific ones. Current valid optimizations: {}".format(" ".join(map(lambda c: c.value, PerformanceFeature)))) +parser.add_argument("--debug-hang", action="store_true", help="Enable stack trace dumps on Ctrl-C for debugging hangs.") + parser.add_argument("--disable-pinned-memory", action="store_true", help="Disable pinned memory use.") parser.add_argument("--mmap-torch-files", action="store_true", help="Use mmap when loading ckpt/pt files.") @@ -221,6 +226,7 @@ parser.add_argument( ) parser.add_argument("--user-directory", type=is_valid_directory, default=None, help="Set the ComfyUI user directory with an absolute path. Overrides --base-directory.") +parser.add_argument("--models-directory", type=is_valid_directory, default=None, help="Set the ComfyUI models directory. Overrides the models folder in --base-directory.") parser.add_argument("--enable-compress-response-body", action="store_true", help="Enable compressing response body.") @@ -236,6 +242,7 @@ database_default_path = os.path.abspath( ) parser.add_argument("--database-url", type=str, default=f"sqlite:///{database_default_path}", help="Specify the database URL, e.g. for an in-memory database you can use 'sqlite:///:memory:'.") parser.add_argument("--enable-assets", action="store_true", help="Enable the assets system (API routes, database synchronization, and background scanning).") +parser.add_argument("--enable-asset-hashing", action="store_true", help="Compute blake3 content hashes when scanning assets. Hashing enables future asset-portability features (deduplication, cross-machine model resolution) but adds startup cost and per-output cost on large models directories. Off by default; enable to opt in.") parser.add_argument("--feature-flag", type=str, action='append', default=[], metavar="KEY[=VALUE]", help="Set a server feature flag. Use KEY=VALUE to set an explicit value, or bare KEY to set it to true. Can be specified multiple times. Boolean values (true/false) and numbers are auto-converted. Examples: --feature-flag show_signin_button=true or --feature-flag show_signin_button") parser.add_argument("--list-feature-flags", action="store_true", help="Print the registry of known CLI-settable feature flags as JSON and exit.") @@ -247,6 +254,9 @@ else: if args.cache_ram is not None and len(args.cache_ram) > 2: parser.error("--cache-ram accepts at most two values: active GB and inactive GB") +if args.high_ram: + args.cache_classic = True + if args.windows_standalone_build: args.auto_launch = True @@ -256,6 +266,10 @@ if args.disable_auto_launch: if args.force_fp16: args.fp16_unet = True +# '--enable-manager-legacy-ui' is meaningless unless the manager is enabled, so imply '--enable-manager'. +if args.enable_manager_legacy_ui: + args.enable_manager = True + # '--fast' is not provided, use an empty set if args.fast is None: diff --git a/comfy/comfy_api_env.py b/comfy/comfy_api_env.py new file mode 100644 index 000000000..17b47933f --- /dev/null +++ b/comfy/comfy_api_env.py @@ -0,0 +1,46 @@ +"""Runtime config the frontend reads from /features to follow --comfy-api-base. + +For a non-prod comfy.org backend (staging or an ephemeral preview env), "/features" exposes the api and +platform base so the frontend talks to it without a rebuild, plus the Firebase environment it should use. +Prod bases are left alone and keep their build-time defaults. +""" + +from typing import Any +from urllib.parse import urlparse + +from comfy.cli_args import args + +_STAGING_API_HOST = "stagingapi.comfy.org" +_TESTENV_HOST_SUFFIX = ".testenvs.comfy.org" +_STAGING_PLATFORM_BASE_URL = "https://stagingplatform.comfy.org" + + +def _is_staging_tier(host: str) -> bool: + return host == _STAGING_API_HOST or host.endswith(_TESTENV_HOST_SUFFIX) + + +def normalize_comfy_api_base(url: str) -> str: + """Rewrite a testenv's friendly main host to its comfy-api '-registry' sibling.""" + parsed = urlparse(url) + host = parsed.hostname or "" + if not host.endswith(_TESTENV_HOST_SUFFIX): + return url + label = host[: -len(_TESTENV_HOST_SUFFIX)] + if label.endswith("-registry"): + return url + return f"{parsed.scheme or 'https'}://{label}-registry{_TESTENV_HOST_SUFFIX}" + + +def environment_overrides_for_base(base_url: str) -> dict[str, Any] | None: + """The /features overrides for a staging-tier base, or None for prod.""" + if not _is_staging_tier(urlparse(base_url).hostname or ""): + return None + return { + "comfy_api_base_url": normalize_comfy_api_base(base_url).rstrip("/"), + "comfy_platform_base_url": _STAGING_PLATFORM_BASE_URL, + "firebase_env": "dev", + } + + +def get_environment_overrides() -> dict[str, Any] | None: + return environment_overrides_for_base(getattr(args, "comfy_api_base", "") or "") diff --git a/comfy/context_windows.py b/comfy/context_windows.py index db57537a2..5f9899c67 100644 --- a/comfy/context_windows.py +++ b/comfy/context_windows.py @@ -8,6 +8,8 @@ from abc import ABC, abstractmethod import logging import comfy.model_management import comfy.patcher_extension +import comfy.utils +import comfy.conds if TYPE_CHECKING: from comfy.model_base import BaseModel from comfy.model_patcher import ModelPatcher @@ -51,12 +53,18 @@ class ContextHandlerABC(ABC): class IndexListContextWindow(ContextWindowABC): - def __init__(self, index_list: list[int], dim: int=0, total_frames: int=0): + def __init__(self, index_list: list[int], dim: int=0, total_frames: int=0, modality_windows: dict=None, context_overlap: int=0): self.index_list = index_list self.context_length = len(index_list) + self.context_overlap = context_overlap self.dim = dim self.total_frames = total_frames self.center_ratio = (min(index_list) + max(index_list)) / (2 * total_frames) + self.modality_windows = modality_windows # dict of {mod_idx: IndexListContextWindow} + self.guide_frames_indices: list[int] = [] + self.guide_overlap_info: list[tuple[int, int]] = [] + self.guide_kf_local_positions: list[int] = [] + self.guide_downscale_factors: list[int] = [] def get_tensor(self, full: torch.Tensor, device=None, dim=None, retain_index_list=[]) -> torch.Tensor: if dim is None: @@ -85,6 +93,11 @@ class IndexListContextWindow(ContextWindowABC): region_idx = int(self.center_ratio * num_regions) return min(max(region_idx, 0), num_regions - 1) + def get_window_for_modality(self, modality_idx: int) -> 'IndexListContextWindow': + if modality_idx == 0: + return self + return self.modality_windows[modality_idx] + class IndexListCallbacks: EVALUATE_CONTEXT_WINDOWS = "evaluate_context_windows" @@ -148,6 +161,172 @@ def slice_cond(cond_value, window: IndexListContextWindow, x_in: torch.Tensor, d return cond_value._copy_with(sliced) +def compute_guide_overlap(guide_entries: list[dict], keyframe_idxs: torch.Tensor, temporal_downscale_ratio: int, window_index_list: list[int]): + """Compute which concatenated guide frames overlap with a context window. + + Each guide's latent-space start is derived from its first token's pixel-t-start + in keyframe_idxs (shape (B, [t,h,w], num_tokens, [start, end])), divided by the + model's temporal_downscale_ratio. + + Args: + guide_entries: list of guide_attention_entry dicts + keyframe_idxs: per-token pixel coords cond tensor for the modality + temporal_downscale_ratio: model's pixel-to-latent temporal compression ratio + window_index_list: the window's frame indices into the video portion + + Returns: + suffix_indices: indices into the guide_frames tensor for frame selection + overlap_info: list of (entry_idx, overlap_count) for guide_attention_entries adjustment + kf_local_positions: window-local frame positions for keyframe_idxs regeneration + total_overlap: total number of overlapping guide frames + """ + window_set = set(window_index_list) + window_list = list(window_index_list) + suffix_indices = [] + overlap_info = [] + kf_local_positions = [] + suffix_base = 0 + token_offset = 0 + + for entry_idx, entry in enumerate(guide_entries): + first_t_pixel = int(keyframe_idxs[0, 0, token_offset, 0].item()) + latent_start = (first_t_pixel + temporal_downscale_ratio - 1) // temporal_downscale_ratio + guide_len = entry["latent_shape"][0] + entry_overlap = 0 + + for local_offset in range(guide_len): + video_pos = latent_start + local_offset + if video_pos in window_set: + suffix_indices.append(suffix_base + local_offset) + kf_local_positions.append(window_list.index(video_pos)) + entry_overlap += 1 + + if entry_overlap > 0: + overlap_info.append((entry_idx, entry_overlap)) + suffix_base += guide_len + token_offset += entry["pre_filter_count"] + + return suffix_indices, overlap_info, kf_local_positions, len(suffix_indices) + + +@dataclass +class WindowingState: + """Per-modality context windowing state for each step, + built using IndexListContextHandler._build_window_state(). + For non-multimodal models the lists are length 1 + """ + latents: list[torch.Tensor] # per-modality working latents (guide frames stripped) + guide_latents: list[torch.Tensor | None] # per-modality guide frames stripped from latents + guide_entries: list[list[dict] | None] # per-modality guide_attention_entry metadata + keyframe_idxs: list[torch.Tensor | None] # per-modality keyframe_idxs tensor for guide latent_start derivation + latent_shapes: list | None # original packed shapes for unpack/pack (None if not multimodal) + dim: int = 0 # primary modality temporal dim for context windowing + is_multimodal: bool = False + temporal_downscale_ratio: int = 1 # model's pixel-to-latent temporal compression ratio + + def prepare_window(self, window: IndexListContextWindow, model) -> IndexListContextWindow: + """Reformat window for multimodal contexts by deriving per-modality index lists. + Non-multimodal contexts return the input window unchanged. + """ + if not self.is_multimodal: + return window + + x = self.latents[0] + primary_total = self.latent_shapes[0][self.dim] + primary_overlap = window.context_overlap + map_shapes = self.latent_shapes + if x.size(self.dim) != primary_total: + map_shapes = list(self.latent_shapes) + video_shape = list(self.latent_shapes[0]) + video_shape[self.dim] = x.size(self.dim) + map_shapes[0] = torch.Size(video_shape) + try: + per_modality_indices = model.map_context_window_to_modalities( + window.index_list, map_shapes, self.dim) + except AttributeError: + raise NotImplementedError( + f"{type(model).__name__} must implement map_context_window_to_modalities for multimodal context windows.") + modality_windows = {} + for mod_idx in range(1, len(self.latents)): + modality_total_frames = self.latents[mod_idx].shape[self.dim] + ratio = modality_total_frames / primary_total if primary_total > 0 else 1 + modality_overlap = max(round(primary_overlap * ratio), 0) + modality_windows[mod_idx] = IndexListContextWindow( + per_modality_indices[mod_idx], dim=self.dim, + total_frames=modality_total_frames, + context_overlap=modality_overlap) + return IndexListContextWindow( + window.index_list, dim=self.dim, total_frames=x.shape[self.dim], + modality_windows=modality_windows, context_overlap=primary_overlap) + + def slice_for_window(self, window: IndexListContextWindow, retain_index_list: list[int], device=None) -> tuple[list[torch.Tensor], list[int]]: + """Slice latents for a context window, injecting guide frames where applicable. + For multimodal contexts, uses the modality-specific windows derived in prepare_window(). + """ + sliced = [] + guide_frame_counts = [] + for idx in range(len(self.latents)): + modality_window = window.get_window_for_modality(idx) + retain = retain_index_list if idx == 0 else [] + s = modality_window.get_tensor(self.latents[idx], device, retain_index_list=retain) + if self.guide_entries[idx] is not None: + s, ng = self._inject_guide_frames(s, modality_window, modality_idx=idx) + else: + ng = 0 + sliced.append(s) + guide_frame_counts.append(ng) + return sliced, guide_frame_counts + + def strip_guide_frames(self, out_per_modality: list[list[torch.Tensor]], guide_frame_counts: list[int], window: IndexListContextWindow): + """Strip injected guide frames from per-cond, per-modality outputs in place.""" + for idx in range(len(self.latents)): + if guide_frame_counts[idx] > 0: + window_len = len(window.get_window_for_modality(idx).index_list) + for ci in range(len(out_per_modality)): + out_per_modality[ci][idx] = out_per_modality[ci][idx].narrow(self.dim, 0, window_len) + + def _inject_guide_frames(self, latent_slice: torch.Tensor, window: IndexListContextWindow, modality_idx: int = 0) -> tuple[torch.Tensor, int]: + guide_entries = self.guide_entries[modality_idx] + guide_frames = self.guide_latents[modality_idx] + keyframe_idxs = self.keyframe_idxs[modality_idx] + suffix_idx, overlap_info, kf_local_pos, guide_frame_count = compute_guide_overlap( + guide_entries, keyframe_idxs, self.temporal_downscale_ratio, window.index_list) + # Shift keyframe positions to account for causal_window_fix anchor occupying sub-pos 0. + anchor_idx = getattr(window, 'causal_anchor_index', None) + if anchor_idx is not None and anchor_idx >= 0: + kf_local_pos = [p + 1 for p in kf_local_pos] + window.guide_frames_indices = suffix_idx + window.guide_overlap_info = overlap_info + window.guide_kf_local_positions = kf_local_pos + + # Derive per-overlap-entry latent_downscale_factor from guide entry latent_shape vs guide frame spatial dims. + # guide_frames has full (post-dilation) spatial dims; entry["latent_shape"] has pre-dilation dims. + guide_downscale_factors = [] + if guide_frame_count > 0: + full_H = guide_frames.shape[3] + for entry_idx, _ in overlap_info: + entry_H = guide_entries[entry_idx]["latent_shape"][1] + guide_downscale_factors.append(full_H // entry_H) + window.guide_downscale_factors = guide_downscale_factors + + if guide_frame_count > 0: + idx = tuple([slice(None)] * self.dim + [suffix_idx]) + return torch.cat([latent_slice, guide_frames[idx]], dim=self.dim), guide_frame_count + return latent_slice, 0 + + def patch_latent_shapes(self, sub_conds, new_shapes): + if not self.is_multimodal: + return + + for cond_list in sub_conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + if 'latent_shapes' in model_conds: + model_conds['latent_shapes'] = comfy.conds.CONDConstant(new_shapes) + + @dataclass class ContextSchedule: name: str @@ -162,7 +341,7 @@ ContextResults = collections.namedtuple("ContextResults", ['window_idx', 'sub_co class IndexListContextHandler(ContextHandlerABC): def __init__(self, context_schedule: ContextSchedule, fuse_method: ContextFuseMethod, context_length: int=1, context_overlap: int=0, context_stride: int=1, closed_loop: bool=False, dim:int=0, freenoise: bool=False, cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, - causal_window_fix: bool=True): + latent_retain_index_list: list[int]=[], causal_window_fix: bool=True): self.context_schedule = context_schedule self.fuse_method = fuse_method self.context_length = context_length @@ -174,17 +353,118 @@ class IndexListContextHandler(ContextHandlerABC): self.freenoise = freenoise self.cond_retain_index_list = [int(x.strip()) for x in cond_retain_index_list.split(",")] if cond_retain_index_list else [] self.split_conds_to_windows = split_conds_to_windows + self.latent_retain_index_list = [int(x.strip()) for x in latent_retain_index_list.split(",")] if latent_retain_index_list else [] self.causal_window_fix = causal_window_fix self.callbacks = {} + @staticmethod + def _get_latent_shapes(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + if 'latent_shapes' in model_conds: + return model_conds['latent_shapes'].cond + return None + + @staticmethod + def _get_guide_entries(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + entries = model_conds.get('guide_attention_entries') + if entries is not None and hasattr(entries, 'cond') and entries.cond: + return entries.cond + return None + + @staticmethod + def _get_keyframe_idxs(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + kf = model_conds.get('keyframe_idxs') + if kf is not None and hasattr(kf, 'cond') and kf.cond is not None: + return kf.cond + return None + + def _apply_freenoise(self, noise: torch.Tensor, conds: list[list[dict]], seed: int) -> torch.Tensor: + """Apply FreeNoise shuffling, scaling context length/overlap per-modality by frame ratio. + If guide frames are present on the primary modality, only the video portion is shuffled. + """ + guide_entries = self._get_guide_entries(conds) + guide_count = sum(e["latent_shape"][0] for e in guide_entries) if guide_entries else 0 + + latent_shapes = self._get_latent_shapes(conds) + if latent_shapes is not None and len(latent_shapes) > 1: + modalities = comfy.utils.unpack_latents(noise, latent_shapes) + primary_total = latent_shapes[0][self.dim] + primary_video_count = modalities[0].size(self.dim) - guide_count + apply_freenoise(modalities[0].narrow(self.dim, 0, primary_video_count), self.dim, self.context_length, self.context_overlap, seed) + for i in range(1, len(modalities)): + mod_total = latent_shapes[i][self.dim] + ratio = mod_total / primary_total if primary_total > 0 else 1 + mod_ctx_len = max(round(self.context_length * ratio), 1) + mod_ctx_overlap = max(round(self.context_overlap * ratio), 0) + modalities[i] = apply_freenoise(modalities[i], self.dim, mod_ctx_len, mod_ctx_overlap, seed) + noise, _ = comfy.utils.pack_latents(modalities) + return noise + video_count = noise.size(self.dim) - guide_count + apply_freenoise(noise.narrow(self.dim, 0, video_count), self.dim, self.context_length, self.context_overlap, seed) + return noise + + def _build_window_state(self, x_in: torch.Tensor, conds: list[list[dict]], model: BaseModel) -> WindowingState: + """Build windowing state for the current step, including unpacking latents and extracting guide frame info from conds.""" + latent_shapes = self._get_latent_shapes(conds) + is_multimodal = latent_shapes is not None and len(latent_shapes) > 1 + unpacked_latents = comfy.utils.unpack_latents(x_in, latent_shapes) if is_multimodal else [x_in] + + unpacked_latents_list = list(unpacked_latents) + guide_latents_list = [None] * len(unpacked_latents) + guide_entries_list = [None] * len(unpacked_latents) + keyframe_idxs_list = [None] * len(unpacked_latents) + + extracted_guide_entries = self._get_guide_entries(conds) + extracted_keyframe_idxs = self._get_keyframe_idxs(conds) + + # Strip guide frames (only from first modality for now) + if extracted_guide_entries is not None: + guide_count = sum(e["latent_shape"][0] for e in extracted_guide_entries) + if guide_count > 0: + x = unpacked_latents[0] + latent_count = x.size(self.dim) - guide_count + unpacked_latents_list[0] = x.narrow(self.dim, 0, latent_count) + guide_latents_list[0] = x.narrow(self.dim, latent_count, guide_count) + guide_entries_list[0] = extracted_guide_entries + keyframe_idxs_list[0] = extracted_keyframe_idxs + + + return WindowingState( + latents=unpacked_latents_list, + guide_latents=guide_latents_list, + guide_entries=guide_entries_list, + keyframe_idxs=keyframe_idxs_list, + latent_shapes=latent_shapes, + dim=self.dim, + is_multimodal=is_multimodal, + temporal_downscale_ratio=model.latent_format.temporal_downscale_ratio) + def should_use_context(self, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]) -> bool: - # for now, assume first dim is batch - should have stored on BaseModel in actual implementation - if x_in.size(self.dim) > self.context_length: - logging.info(f"Using context windows {self.context_length} with overlap {self.context_overlap} for {x_in.size(self.dim)} frames.") + window_state = self._build_window_state(x_in, conds, model) # build window_state to check frame counts, will be built again in execute + total_frame_count = window_state.latents[0].size(self.dim) + if total_frame_count > self.context_length: + logging.info(f"\nUsing context windows: Context length {self.context_length} with overlap {self.context_overlap} for {total_frame_count} frames.") if self.cond_retain_index_list: logging.info(f"Retaining original cond for indexes: {self.cond_retain_index_list}") + if self.latent_retain_index_list: + logging.info(f"Retaining original latent for indexes: {self.latent_retain_index_list}") return True + logging.info(f"\nNot using context windows since context length ({self.context_length}) exceeds input frames ({total_frame_count}).") return False def prepare_control_objects(self, control: ControlBase, device=None) -> ControlBase: @@ -275,7 +555,9 @@ class IndexListContextHandler(ContextHandlerABC): return resized_cond def set_step(self, timestep: torch.Tensor, model_options: dict[str]): - mask = torch.isclose(model_options["transformer_options"]["sample_sigmas"], timestep[0], rtol=0.0001) + sample_sigmas = model_options["transformer_options"]["sample_sigmas"] + current_timestep = timestep[0].to(sample_sigmas.dtype) + mask = torch.isclose(sample_sigmas, current_timestep, rtol=0.0001) matches = torch.nonzero(mask) if torch.numel(matches) == 0: return # substep from multi-step sampler: keep self._step from the last full step @@ -284,54 +566,98 @@ class IndexListContextHandler(ContextHandlerABC): def get_context_windows(self, model: BaseModel, x_in: torch.Tensor, model_options: dict[str]) -> list[IndexListContextWindow]: full_length = x_in.size(self.dim) # TODO: choose dim based on model context_windows = self.context_schedule.func(full_length, self, model_options) - context_windows = [IndexListContextWindow(window, dim=self.dim, total_frames=full_length) for window in context_windows] + context_windows = [IndexListContextWindow(window, dim=self.dim, total_frames=full_length, context_overlap=self.context_overlap) for window in context_windows] return context_windows def execute(self, calc_cond_batch: Callable, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]): self._model = model self.set_step(timestep, model_options) - context_windows = self.get_context_windows(model, x_in, model_options) - enumerated_context_windows = list(enumerate(context_windows)) - conds_final = [torch.zeros_like(x_in) for _ in conds] + window_state = self._build_window_state(x_in, conds, model) + num_modalities = len(window_state.latents) + + context_windows = self.get_context_windows(model, window_state.latents[0], model_options) + enumerated_context_windows = list(enumerate(context_windows)) + total_windows = len(enumerated_context_windows) + + # Initialize per-modality accumulators (length 1 for single-modality) + accum = [[torch.zeros_like(m) for _ in conds] for m in window_state.latents] if self.fuse_method.name == ContextFuseMethods.RELATIVE: - counts_final = [torch.ones(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds] + counts = [[torch.ones(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents] else: - counts_final = [torch.zeros(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds] - biases_final = [([0.0] * x_in.shape[self.dim]) for _ in conds] + counts = [[torch.zeros(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents] + biases = [[([0.0] * m.shape[self.dim]) for _ in conds] for m in window_state.latents] for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_START, self.callbacks): callback(self, model, x_in, conds, timestep, model_options) + # accumulate results from each context window for enum_window in enumerated_context_windows: - results = self.evaluate_context_windows(calc_cond_batch, model, x_in, conds, timestep, [enum_window], model_options) + results = self.evaluate_context_windows( + calc_cond_batch, model, x_in, conds, timestep, [enum_window], + model_options, window_state=window_state, total_windows=total_windows) for result in results: - self.combine_context_window_results(x_in, result.sub_conds_out, result.sub_conds, result.window, result.window_idx, len(enumerated_context_windows), timestep, - conds_final, counts_final, biases_final) + # result.sub_conds_out is per-cond, per-modality: list[list[Tensor]] + for mod_idx in range(num_modalities): + mod_out = [result.sub_conds_out[ci][mod_idx] for ci in range(len(conds))] + modality_window = result.window.get_window_for_modality(mod_idx) + self.combine_context_window_results( + window_state.latents[mod_idx], mod_out, result.sub_conds, modality_window, + result.window_idx, total_windows, timestep, + accum[mod_idx], counts[mod_idx], biases[mod_idx]) + + # fuse accumulated results into final conds try: - # finalize conds - if self.fuse_method.name == ContextFuseMethods.RELATIVE: - # relative is already normalized, so return as is - del counts_final - return conds_final - else: - # normalize conds via division by context usage counts - for i in range(len(conds_final)): - conds_final[i] /= counts_final[i] - del counts_final - return conds_final + result_out = [] + for ci in range(len(conds)): + finalized = [] + for mod_idx in range(num_modalities): + if self.fuse_method.name != ContextFuseMethods.RELATIVE: + accum[mod_idx][ci] /= counts[mod_idx][ci] + f = accum[mod_idx][ci] + + # if guide frames were injected, append them to the end of the fused latents for the next step + if window_state.guide_latents[mod_idx] is not None: + f = torch.cat([f, window_state.guide_latents[mod_idx]], dim=self.dim) + finalized.append(f) + + # pack modalities together if needed + if window_state.is_multimodal and len(finalized) > 1: + packed, _ = comfy.utils.pack_latents(finalized) + else: + packed = finalized[0] + + result_out.append(packed) + return result_out finally: for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_CLEANUP, self.callbacks): callback(self, model, x_in, conds, timestep, model_options) - def evaluate_context_windows(self, calc_cond_batch: Callable, model: BaseModel, x_in: torch.Tensor, conds, timestep: torch.Tensor, enumerated_context_windows: list[tuple[int, IndexListContextWindow]], - model_options, device=None, first_device=None): + def evaluate_context_windows(self, calc_cond_batch: Callable, model: BaseModel, x_in: torch.Tensor, conds, + timestep: torch.Tensor, enumerated_context_windows: list[tuple[int, IndexListContextWindow]], + model_options, window_state: WindowingState, total_windows: int = None, + device=None, first_device=None): + """Evaluate context windows and return per-cond, per-modality outputs in ContextResults.sub_conds_out + + For each window: + 1. Builds windows (for each modality if multimodal) + 2. Slices window for each modality + 3. Injects concatenated latent guide frames where present + 4. Packs together if needed and calls model + 5. Unpacks and strips any guides from outputs + """ + x = window_state.latents[0] + results: list[ContextResults] = [] for window_idx, window in enumerated_context_windows: # allow processing to end between context window executions for faster Cancel comfy.model_management.throw_exception_if_processing_interrupted() - # causal_window_fix: prepend a pre-window frame that will be stripped post-forward + # prepare the window accounting for multimodal windows + window = window_state.prepare_window(window, model) + + # causal_window_fix: prepend a pre-window frame that will be stripped post-forward. + # Set anchor before slice_for_window so the latent slice and downstream cond slices both pick it up. anchor_applied = False if self.causal_window_fix: anchor_idx = window.index_list[0] - 1 @@ -339,27 +665,46 @@ class IndexListContextHandler(ContextHandlerABC): window.causal_anchor_index = anchor_idx anchor_applied = True + # slice the window for each modality, injecting guide frames where applicable + sliced, guide_frame_counts_per_modality = window_state.slice_for_window(window, self.latent_retain_index_list, device) + for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EVALUATE_CONTEXT_WINDOWS, self.callbacks): callback(self, model, x_in, conds, timestep, model_options, window_idx, window, model_options, device, first_device) - # update exposed params + logging.info(f"Context window {window_idx + 1}/{total_windows or len(enumerated_context_windows)}: frames {window.index_list[0]}-{window.index_list[-1]} of {x.shape[self.dim]}" + + (f" (+{guide_frame_counts_per_modality[0]} guide frames)" if guide_frame_counts_per_modality[0] > 0 else "") + ) + + # if multimodal, pack modalities together + if window_state.is_multimodal and len(sliced) > 1: + sub_x, sub_shapes = comfy.utils.pack_latents(sliced) + else: + sub_x, sub_shapes = sliced[0], [sliced[0].shape] + + # get resized conds for window model_options["transformer_options"]["context_window"] = window - # get subsections of x, timestep, conds - sub_x = window.get_tensor(x_in, device) - sub_timestep = window.get_tensor(timestep, device, dim=0) - sub_conds = [self.get_resized_cond(cond, x_in, window, device) for cond in conds] + sub_timestep = window.get_tensor(timestep, dim=0) + sub_conds = [self.get_resized_cond(cond, x, window) for cond in conds] + # if multimodal, patch latent_shapes in conds for correct unpacking in model + window_state.patch_latent_shapes(sub_conds, sub_shapes) + + # call model on window sub_conds_out = calc_cond_batch(model, sub_conds, sub_x, sub_timestep, model_options) - if device is not None: - for i in range(len(sub_conds_out)): - sub_conds_out[i] = sub_conds_out[i].to(x_in.device) - # strip causal_window_fix anchor if applied + # unpack outputs + out_per_modality = [comfy.utils.unpack_latents(sub_conds_out[i], sub_shapes) for i in range(len(sub_conds_out))] + + # strip causal_window_fix anchor from primary modality before guide strip so window_len math stays correct if anchor_applied: - for i in range(len(sub_conds_out)): - sub_conds_out[i] = sub_conds_out[i].narrow(self.dim, 1, sub_conds_out[i].shape[self.dim] - 1) + for ci in range(len(out_per_modality)): + t = out_per_modality[ci][0] + out_per_modality[ci][0] = t.narrow(self.dim, 1, t.shape[self.dim] - 1) - results.append(ContextResults(window_idx, sub_conds_out, sub_conds, window)) + # strip injected guide frames + window_state.strip_guide_frames(out_per_modality, guide_frame_counts_per_modality, window) + + results.append(ContextResults(window_idx, out_per_modality, sub_conds, window)) return results @@ -383,7 +728,7 @@ class IndexListContextHandler(ContextHandlerABC): biases_final[i][idx] = bias_total + bias else: # add conds and counts based on weights of fuse method - weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep) + weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep, context_overlap=window.context_overlap) weights_tensor = match_weights_to_dim(weights, x_in, self.dim, device=x_in.device) for i in range(len(sub_conds_out)): window.add_window(conds_final[i], sub_conds_out[i] * weights_tensor) @@ -393,16 +738,22 @@ class IndexListContextHandler(ContextHandlerABC): callback(self, x_in, sub_conds_out, sub_conds, window, window_idx, total_windows, timestep, conds_final, counts_final, biases_final) -def _prepare_sampling_wrapper(executor, model, noise_shape: torch.Tensor, *args, **kwargs): - # limit noise_shape length to context_length for more accurate vram use estimation +def _prepare_sampling_wrapper(executor, model, noise_shape: torch.Tensor, conds, *args, **kwargs): + # Scale noise_shape to a single context window so VRAM estimation budgets per-window. model_options = kwargs.get("model_options", None) if model_options is None: raise Exception("model_options not found in prepare_sampling_wrapper; this should never happen, something went wrong.") handler: IndexListContextHandler = model_options.get("context_handler", None) if handler is not None: noise_shape = list(noise_shape) - noise_shape[handler.dim] = min(noise_shape[handler.dim], handler.context_length) - return executor(model, noise_shape, *args, **kwargs) + is_packed = len(noise_shape) == 3 and noise_shape[1] == 1 + if is_packed: + # TODO: latent_shapes cond isn't attached yet at this point, so we can't compute a + # per-window flat latent here. Skipping the clamp over-estimates but prevents immediate OOM. + pass + elif handler.dim < len(noise_shape) and noise_shape[handler.dim] > handler.context_length: + noise_shape[handler.dim] = min(noise_shape[handler.dim], handler.context_length) + return executor(model, noise_shape, conds, *args, **kwargs) def create_prepare_sampling_wrapper(model: ModelPatcher): @@ -422,11 +773,12 @@ def _sampler_sample_wrapper(executor, guider, sigmas, extra_args, callback, nois raise Exception("context_handler not found in sampler_sample_wrapper; this should never happen, something went wrong.") if not handler.freenoise: return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs) - noise = apply_freenoise(noise, handler.dim, handler.context_length, handler.context_overlap, extra_args["seed"]) + + conds = [guider.conds.get('positive', guider.conds.get('negative', []))] + noise = handler._apply_freenoise(noise, conds, extra_args["seed"]) return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs) - def create_sampler_sample_wrapper(model: ModelPatcher): model.add_wrapper_with_key( comfy.patcher_extension.WrappersMP.SAMPLER_SAMPLE, @@ -434,7 +786,6 @@ def create_sampler_sample_wrapper(model: ModelPatcher): _sampler_sample_wrapper ) - def match_weights_to_dim(weights: list[float], x_in: torch.Tensor, dim: int, device=None) -> torch.Tensor: total_dims = len(x_in.shape) weights_tensor = torch.Tensor(weights).to(device=device) @@ -580,8 +931,9 @@ def get_matching_context_schedule(context_schedule: str) -> ContextSchedule: return ContextSchedule(context_schedule, func) -def get_context_weights(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, sigma: torch.Tensor=None): - return handler.fuse_method.func(length, sigma=sigma, handler=handler, full_length=full_length, idxs=idxs) +def get_context_weights(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, sigma: torch.Tensor=None, context_overlap: int=None): + context_overlap = handler.context_overlap if context_overlap is None else context_overlap + return handler.fuse_method.func(length, sigma=sigma, handler=handler, full_length=full_length, idxs=idxs, context_overlap=context_overlap) def create_weights_flat(length: int, **kwargs) -> list[float]: @@ -599,18 +951,18 @@ def create_weights_pyramid(length: int, **kwargs) -> list[float]: weight_sequence = list(range(1, max_weight, 1)) + [max_weight] + list(range(max_weight - 1, 0, -1)) return weight_sequence -def create_weights_overlap_linear(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, **kwargs): +def create_weights_overlap_linear(length: int, full_length: int, idxs: list[int], context_overlap: int, **kwargs): # based on code in Kijai's WanVideoWrapper: https://github.com/kijai/ComfyUI-WanVideoWrapper/blob/dbb2523b37e4ccdf45127e5ae33e31362f755c8e/nodes.py#L1302 # only expected overlap is given different weights weights_torch = torch.ones((length)) # blend left-side on all except first window if min(idxs) > 0: - ramp_up = torch.linspace(1e-37, 1, handler.context_overlap) - weights_torch[:handler.context_overlap] = ramp_up + ramp_up = torch.linspace(1e-37, 1, context_overlap) + weights_torch[:context_overlap] = ramp_up # blend right-side on all except last window if max(idxs) < full_length-1: - ramp_down = torch.linspace(1, 1e-37, handler.context_overlap) - weights_torch[-handler.context_overlap:] = ramp_down + ramp_down = torch.linspace(1, 1e-37, context_overlap) + weights_torch[-context_overlap:] = ramp_down return weights_torch class ContextFuseMethods: diff --git a/comfy/image_encoders/dino2.py b/comfy/image_encoders/dino2.py index ee86f8309..53e4fdb6c 100644 --- a/comfy/image_encoders/dino2.py +++ b/comfy/image_encoders/dino2.py @@ -1,7 +1,13 @@ import torch +import torch.nn.functional as F + from comfy.text_encoders.bert import BertAttention import comfy.model_management from comfy.ldm.modules.attention import optimized_attention_for_device +from comfy.ldm.depth_anything_3.reference_view_selector import ( + select_reference_view, reorder_by_reference, restore_original_order, + THRESH_FOR_REF_SELECTION, +) class Dino2AttentionOutput(torch.nn.Module): @@ -14,13 +20,41 @@ class Dino2AttentionOutput(torch.nn.Module): class Dino2AttentionBlock(torch.nn.Module): - def __init__(self, embed_dim, heads, layer_norm_eps, dtype, device, operations): + def __init__(self, embed_dim, heads, layer_norm_eps, dtype, device, operations, + qk_norm=False): super().__init__() + self.heads = heads + self.head_dim = embed_dim // heads self.attention = BertAttention(embed_dim, heads, dtype, device, operations) self.output = Dino2AttentionOutput(embed_dim, embed_dim, layer_norm_eps, dtype, device, operations) + if qk_norm: + self.q_norm = operations.LayerNorm(self.head_dim, dtype=dtype, device=device) + self.k_norm = operations.LayerNorm(self.head_dim, dtype=dtype, device=device) + else: + self.q_norm = None + self.k_norm = None - def forward(self, x, mask, optimized_attention): - return self.output(self.attention(x, mask, optimized_attention)) + def forward(self, x, mask, optimized_attention, pos=None, rope=None): + # Fast path used by the existing CLIP-vision DINOv2 (no DA3 extensions). + if self.q_norm is None and rope is None: + return self.output(self.attention(x, mask, optimized_attention)) + + # DA3 path: do QKV manually so we can apply per-head QK-norm and 2D RoPE. + attn = self.attention + B, N, C = x.shape + h = self.heads + d = self.head_dim + q = attn.query(x).view(B, N, h, d).transpose(1, 2) + k = attn.key(x).view(B, N, h, d).transpose(1, 2) + v = attn.value(x).view(B, N, h, d).transpose(1, 2) + if self.q_norm is not None: + q = self.q_norm(q) + k = self.k_norm(k) + if rope is not None and pos is not None: + q = rope(q, pos) + k = rope(k, pos) + out = optimized_attention(q, k, v, h, mask=mask, skip_reshape=True) + return self.output(out) class LayerScale(torch.nn.Module): @@ -64,9 +98,11 @@ class SwiGLUFFN(torch.nn.Module): class Dino2Block(torch.nn.Module): - def __init__(self, dim, num_heads, layer_norm_eps, dtype, device, operations, use_swiglu_ffn): + def __init__(self, dim, num_heads, layer_norm_eps, dtype, device, operations, use_swiglu_ffn, + qk_norm=False): super().__init__() - self.attention = Dino2AttentionBlock(dim, num_heads, layer_norm_eps, dtype, device, operations) + self.attention = Dino2AttentionBlock(dim, num_heads, layer_norm_eps, dtype, device, operations, + qk_norm=qk_norm) self.layer_scale1 = LayerScale(dim, dtype, device, operations) self.layer_scale2 = LayerScale(dim, dtype, device, operations) if use_swiglu_ffn: @@ -76,19 +112,90 @@ class Dino2Block(torch.nn.Module): self.norm1 = operations.LayerNorm(dim, eps=layer_norm_eps, dtype=dtype, device=device) self.norm2 = operations.LayerNorm(dim, eps=layer_norm_eps, dtype=dtype, device=device) - def forward(self, x, optimized_attention): - x = x + self.layer_scale1(self.attention(self.norm1(x), None, optimized_attention)) + def forward(self, x, optimized_attention, pos=None, rope=None, attn_mask=None): + x = x + self.layer_scale1(self.attention(self.norm1(x), attn_mask, optimized_attention, + pos=pos, rope=rope)) x = x + self.layer_scale2(self.mlp(self.norm2(x))) return x -class Dino2Encoder(torch.nn.Module): - def __init__(self, dim, num_heads, layer_norm_eps, num_layers, dtype, device, operations, use_swiglu_ffn): +# ----------------------------------------------------------------------------- +# 2D Rotary position embedding (DA3 extension) +# ----------------------------------------------------------------------------- + + +class _PositionGetter: + """Cache (h, w) -> flat (y, x) position grid used to feed ``rope``.""" + + def __init__(self): + self._cache: dict = {} + + def __call__(self, batch_size: int, height: int, width: int, device) -> torch.Tensor: + key = (height, width, device) + if key not in self._cache: + y = torch.arange(height, device=device) + x = torch.arange(width, device=device) + self._cache[key] = torch.cartesian_prod(y, x) + cached = self._cache[key] + return cached.view(1, height * width, 2).expand(batch_size, -1, -1).clone() + + +class RotaryPositionEmbedding2D(torch.nn.Module): + """2D RoPE used by DA3-Small/Base. No learnable parameters.""" + + def __init__(self, frequency: float = 100.0): super().__init__() - self.layer = torch.nn.ModuleList([Dino2Block(dim, num_heads, layer_norm_eps, dtype, device, operations, use_swiglu_ffn = use_swiglu_ffn) - for _ in range(num_layers)]) + self.base_frequency = frequency + self._freq_cache: dict = {} + + def _components(self, dim: int, seq_len: int, device, dtype): + key = (dim, seq_len, device, dtype) + if key not in self._freq_cache: + exp = torch.arange(0, dim, 2, device=device).float() / dim + inv_freq = 1.0 / (self.base_frequency ** exp) + pos = torch.arange(seq_len, device=device, dtype=inv_freq.dtype) + ang = torch.einsum("i,j->ij", pos, inv_freq) + ang = ang.to(dtype) + ang = torch.cat((ang, ang), dim=-1) + self._freq_cache[key] = (ang.cos().to(dtype), ang.sin().to(dtype)) + return self._freq_cache[key] + + @staticmethod + def _rotate(x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] + x1, x2 = x[..., : d // 2], x[..., d // 2:] + return torch.cat((-x2, x1), dim=-1) + + def _apply_1d(self, tokens, positions, cos_c, sin_c): + cos = F.embedding(positions, cos_c)[:, None, :, :] + sin = F.embedding(positions, sin_c)[:, None, :, :] + return (tokens * cos) + (self._rotate(tokens) * sin) + + def forward(self, tokens: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + feature_dim = tokens.size(-1) // 2 + max_pos = int(positions.max()) + 1 + cos_c, sin_c = self._components(feature_dim, max_pos, tokens.device, tokens.dtype) + v, h = tokens.chunk(2, dim=-1) + v = self._apply_1d(v, positions[..., 0], cos_c, sin_c) + h = self._apply_1d(h, positions[..., 1], cos_c, sin_c) + return torch.cat((v, h), dim=-1) + + +class Dino2Encoder(torch.nn.Module): + def __init__(self, dim, num_heads, layer_norm_eps, num_layers, dtype, device, operations, use_swiglu_ffn, + qknorm_start: int = -1): + super().__init__() + self.layer = torch.nn.ModuleList([ + Dino2Block( + dim, num_heads, layer_norm_eps, dtype, device, operations, + use_swiglu_ffn=use_swiglu_ffn, + qk_norm=(qknorm_start != -1 and i >= qknorm_start), + ) + for i in range(num_layers) + ]) def forward(self, x, intermediate_output=None): + # Backward-compat path used by ``ClipVisionModel`` (no DA3 extensions). optimized_attention = optimized_attention_for_device(x.device, False, small_input=True) if intermediate_output is not None: @@ -122,16 +229,27 @@ class Dino2PatchEmbeddings(torch.nn.Module): class Dino2Embeddings(torch.nn.Module): - def __init__(self, dim, dtype, device, operations): + def __init__(self, dim, dtype, device, operations, + patch_size: int = 14, image_size: int = 518, + use_mask_token: bool = True, + num_camera_tokens: int = 0): super().__init__() - patch_size = 14 - image_size = 518 self.patch_size = patch_size + self.image_size = image_size self.patch_embeddings = Dino2PatchEmbeddings(dim, patch_size=patch_size, image_size=image_size, dtype=dtype, device=device, operations=operations) self.position_embeddings = torch.nn.Parameter(torch.empty(1, (image_size // patch_size) ** 2 + 1, dim, dtype=dtype, device=device)) self.cls_token = torch.nn.Parameter(torch.empty(1, 1, dim, dtype=dtype, device=device)) # mask_token is a pre-training param, kept only so strict loading accepts the key. - self.mask_token = torch.nn.Parameter(torch.empty(1, dim, dtype=dtype, device=device)) + if use_mask_token: + self.mask_token = torch.nn.Parameter(torch.empty(1, dim, dtype=dtype, device=device)) + else: + self.mask_token = None + if num_camera_tokens > 0: + # DA3 stores (ref_token, src_token) pairs that get injected at the + # alt-attn boundary; see ``Dinov2Model._inject_camera_token``. + self.camera_token = torch.nn.Parameter(torch.empty(1, num_camera_tokens, dim, dtype=dtype, device=device)) + else: + self.camera_token = None def interpolate_pos_encoding(self, x, h_pixels, w_pixels): pos_embed = comfy.model_management.cast_to_device(self.position_embeddings, x.device, torch.float32) @@ -140,12 +258,22 @@ class Dino2Embeddings(torch.nn.Module): patch_pos = pos_embed[:, 1:] N = patch_pos.shape[1] M = int(N ** 0.5) + assert N == M * M, f"DINOv2 position grid must be square, got N={N} patches (sqrt={M})" h0 = h_pixels // self.patch_size w0 = w_pixels // self.patch_size - scale_factor = ((h0 + 0.1) / M, (w0 + 0.1) / M) # +0.1 matches upstream DINOv2's FP-rounding workaround so the interpolate output size lands on (h0, w0). + # +0.1 matches upstream DINOv2's FP-rounding workaround so the interpolate output size lands on (h0, w0). + # scale_factor is (height_scale, width_scale) -- height MUST come first; + # swapping these only happens to work for square inputs and breaks + # non-square paths like DA3-Small / DA3-Base multi-view. + scale_factor = ((h0 + 0.1) / M, (w0 + 0.1) / M) patch_pos = patch_pos.reshape(1, M, M, -1).permute(0, 3, 1, 2) patch_pos = torch.nn.functional.interpolate(patch_pos, scale_factor=scale_factor, mode="bicubic", antialias=False) + assert (h0, w0) == patch_pos.shape[-2:], ( + f"Interpolated pos-embed grid {tuple(patch_pos.shape[-2:])} does not match " + f"target patch grid ({h0}, {w0}) for input {h_pixels}x{w_pixels} (patch_size={self.patch_size}); " + f"check scale_factor axis order and +0.1 rounding workaround" + ) patch_pos = patch_pos.permute(0, 2, 3, 1).flatten(1, 2) return torch.cat((class_pos, patch_pos), dim=1).to(x.dtype) @@ -168,12 +296,51 @@ class Dinov2Model(torch.nn.Module): heads = config_dict["num_attention_heads"] layer_norm_eps = config_dict["layer_norm_eps"] use_swiglu_ffn = config_dict["use_swiglu_ffn"] + patch_size = config_dict.get("patch_size", 14) + image_size = config_dict.get("image_size", 518) + use_mask_token = config_dict.get("use_mask_token", True) - self.embeddings = Dino2Embeddings(dim, dtype, device, operations) - self.encoder = Dino2Encoder(dim, heads, layer_norm_eps, num_layers, dtype, device, operations, use_swiglu_ffn = use_swiglu_ffn) + # DA3 extensions (all default to disabled). + self.alt_start = config_dict.get("alt_start", -1) + self.qknorm_start = config_dict.get("qknorm_start", -1) + self.rope_start = config_dict.get("rope_start", -1) + self.cat_token = config_dict.get("cat_token", False) + rope_freq = config_dict.get("rope_freq", 100.0) + + self.embed_dim = dim + self.patch_size = patch_size + self.num_register_tokens = 0 + self.patch_start_idx = 1 + + if self.rope_start != -1 and rope_freq > 0: + self.rope = RotaryPositionEmbedding2D(frequency=rope_freq) + self._position_getter = _PositionGetter() + else: + self.rope = None + self._position_getter = None + + # camera_token shape: (1, 2, dim) -> (ref_token, src_token). + num_cam_tokens = 2 if self.alt_start != -1 else 0 + + self.embeddings = Dino2Embeddings( + dim, dtype, device, operations, + patch_size=patch_size, image_size=image_size, + use_mask_token=use_mask_token, num_camera_tokens=num_cam_tokens, + ) + self.encoder = Dino2Encoder( + dim, heads, layer_norm_eps, num_layers, dtype, device, operations, + use_swiglu_ffn=use_swiglu_ffn, + qknorm_start=self.qknorm_start, + ) self.layernorm = operations.LayerNorm(dim, eps=layer_norm_eps, dtype=dtype, device=device) def forward(self, pixel_values, attention_mask=None, intermediate_output=None): + if self.alt_start != -1: + raise RuntimeError( + "Dinov2Model.forward() is the backward-compatible CLIP-vision path and does not " + "apply DA3 extensions (RoPE, alternating attention, camera-token injection). " + "Use get_intermediate_layers_da3() for Depth Anything 3 models." + ) x = self.embeddings(pixel_values) x, i = self.encoder(x, intermediate_output=intermediate_output) x = self.layernorm(x) @@ -181,6 +348,7 @@ class Dinov2Model(torch.nn.Module): return x, i, pooled_output, None def get_intermediate_layers(self, pixel_values, indices, apply_norm=True): + """Single-view multi-layer feature extraction.""" x = self.embeddings(pixel_values) optimized_attention = optimized_attention_for_device(x.device, False, small_input=True) n_layers = len(self.encoder.layer) @@ -197,3 +365,132 @@ class Dinov2Model(torch.nn.Module): if i >= max_idx: break return [cache[i] for i in resolved] + + # ------------------------------------------------------------------ + # Depth Anything 3 forward + # ------------------------------------------------------------------ + def _prepare_rope_positions(self, B, S, H, W, device): + if self.rope is None: + return None, None + ph, pw = H // self.patch_size, W // self.patch_size + pos = self._position_getter(B * S, ph, pw, device=device) + # Shift so the cls/cam token at position 0 is reserved for "no diff". + pos = pos + 1 + cls_pos = torch.zeros(B * S, self.patch_start_idx, 2, device=device, dtype=pos.dtype) + # Per-view local: real grid positions for patches, 0 for cls token. + pos_local = torch.cat([cls_pos, pos], dim=1) + # Global (across views): same grid positions; cls token still at 0, + # but patches share the same positions in every view. + pos_global = torch.cat([cls_pos, torch.zeros_like(pos) + 1], dim=1) + return pos_local, pos_global + + def _inject_camera_token(self, x: torch.Tensor, B: int, S: int, cam_token: "torch.Tensor | None") -> torch.Tensor: + # x: (B, S, N, C). Replace token at index 0 with the camera token. + if cam_token is not None: + inj = cam_token + else: + ct = comfy.model_management.cast_to_device(self.embeddings.camera_token, x.device, x.dtype) + ref_token = ct[:, :1].expand(B, -1, -1) + src_token = ct[:, 1:].expand(B, max(S - 1, 0), -1) + inj = torch.cat([ref_token, src_token], dim=1) + x = x.clone() + x[:, :, 0] = inj + return x + + def get_intermediate_layers_da3(self, pixel_values, out_layers, cam_token=None, ref_view_strategy="saddle_balanced", export_feat_layers=None): + """Multi-view multi-layer feature extraction used by Depth Anything 3.""" + if pixel_values.ndim == 4: + pixel_values = pixel_values.unsqueeze(1) + assert pixel_values.ndim == 5 and pixel_values.shape[2] == 3, \ + f"expected (B,3,H,W) or (B,S,3,H,W); got {tuple(pixel_values.shape)}" + B, S, _, H, W = pixel_values.shape + + # Patch + cls + (interpolated) pos embed for each view. + x = pixel_values.reshape(B * S, 3, H, W) + x = self.embeddings(x) # (B*S, 1+N, C) + x = x.reshape(B, S, x.shape[-2], x.shape[-1]) # (B, S, 1+N, C) + + pos_local, pos_global = self._prepare_rope_positions(B, S, H, W, x.device) + # optimized_attention is only used by blocks without QK-norm/RoPE + # (vanilla DINOv2 path); enabling-aware blocks fall through to SDPA. + optimized_attention = optimized_attention_for_device(x.device, False, small_input=True) + + out_set = set(out_layers) + export_set = set(export_feat_layers) if export_feat_layers else set() + outputs: list[torch.Tensor] = [] + aux_outputs: list[torch.Tensor] = [] + local_x = x + b_idx = None + + + for i, blk in enumerate(self.encoder.layer): + apply_rope = self.rope is not None and i >= self.rope_start + block_rope = self.rope if apply_rope else None + l_pos = pos_local if apply_rope else None + g_pos = pos_global if apply_rope else None + + # Reference-view selection threshold: matches the upstream constant + # THRESH_FOR_REF_SELECTION = 3. Skipped when a user-supplied + # cam_token is provided (camera info already pins the geometry). + if (self.alt_start != -1 and i == self.alt_start - 1 and S >= THRESH_FOR_REF_SELECTION and cam_token is None): + b_idx = select_reference_view(x, strategy=ref_view_strategy) + x = reorder_by_reference(x, b_idx) + local_x = reorder_by_reference(local_x, b_idx) + + if self.alt_start != -1 and i == self.alt_start: + x = self._inject_camera_token(x, B, S, cam_token) + + if self.alt_start != -1 and i >= self.alt_start and (i % 2 == 1): + # Global attention across views: flatten S into the seq dim. + t = x.reshape(B, S * x.shape[-2], x.shape[-1]) + p = g_pos.reshape(B, S * g_pos.shape[-2], g_pos.shape[-1]) if g_pos is not None else None + t = blk(t, optimized_attention=optimized_attention, pos=p, rope=block_rope) + x = t.reshape(B, S, x.shape[-2], x.shape[-1]) + else: + # Per-view local attention. + t = x.reshape(B * S, x.shape[-2], x.shape[-1]) + p = l_pos.reshape(B * S, l_pos.shape[-2], l_pos.shape[-1]) if l_pos is not None else None + t = blk(t, optimized_attention=optimized_attention, pos=p, rope=block_rope) + x = t.reshape(B, S, x.shape[-2], x.shape[-1]) + local_x = x + + if i in out_set: + if self.cat_token: + out_x = torch.cat([local_x, x], dim=-1) + else: + out_x = x + # Restore original view order on the way out so heads see views + # in the user's expected order. + if b_idx is not None and self.alt_start != -1: + out_x = restore_original_order(out_x, b_idx) + outputs.append(out_x) + + if i in export_set: + aux = x + if b_idx is not None and self.alt_start != -1: + aux = restore_original_order(aux, b_idx) + aux_outputs.append(aux) + + # Apply final norm. When cat_token is set, only the right half + # ("global" features) is normalised; the left half is left as-is to + # match the upstream DA3 head signature. + normed: list[torch.Tensor] = [] + cls_tokens: list[torch.Tensor] = [] + for out_x in outputs: + cls_tokens.append(out_x[:, :, 0]) + if out_x.shape[-1] == self.embed_dim: + normed.append(self.layernorm(out_x)) + elif out_x.shape[-1] == self.embed_dim * 2: + left = out_x[..., :self.embed_dim] + right = self.layernorm(out_x[..., self.embed_dim:]) + normed.append(torch.cat([left, right], dim=-1)) + else: + raise ValueError(f"Unexpected token width: {out_x.shape[-1]}") + + # Drop cls/cam token from the patch sequence. + normed = [o[..., 1 + self.num_register_tokens:, :] for o in normed] + + # Final layernorm + drop cls token from auxiliary features too. + aux_normed = [self.layernorm(o)[..., 1 + self.num_register_tokens:, :] + for o in aux_outputs] + return list(zip(normed, cls_tokens)), aux_normed diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index bbdfd4bc2..8a16cfe55 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -779,6 +779,10 @@ class ACEAudio(LatentFormat): latent_channels = 8 latent_dimensions = 2 +class SeedVR2(LatentFormat): + latent_channels = 16 + latent_dimensions = 3 + class ACEAudio15(LatentFormat): latent_channels = 64 latent_dimensions = 1 diff --git a/comfy/ldm/ace/ace_step15.py b/comfy/ldm/ace/ace_step15.py index 2ca2d26c4..02182c49f 100644 --- a/comfy/ldm/ace/ace_step15.py +++ b/comfy/ldm/ace/ace_step15.py @@ -217,10 +217,7 @@ class AceStepAttention(nn.Module): cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - n_rep = self.num_heads // self.num_kv_heads - if n_rep > 1: - key_states = key_states.repeat_interleave(n_rep, dim=1) - value_states = value_states.repeat_interleave(n_rep, dim=1) + gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {} attn_bias = None if self.sliding_window is not None and not self.is_cross_attention: @@ -244,7 +241,7 @@ class AceStepAttention(nn.Module): else: attn_bias = window_bias - attn_output = optimized_attention(query_states, key_states, value_states, self.num_heads, attn_bias, skip_reshape=True, low_precision_attention=False) + attn_output = optimized_attention(query_states, key_states, value_states, self.num_heads, attn_bias, skip_reshape=True, low_precision_attention=False, **gqa_kwargs) attn_output = self.o_proj(attn_output) return attn_output diff --git a/comfy/ldm/audio/dit.py b/comfy/ldm/audio/dit.py index c28be5b49..b0759a240 100644 --- a/comfy/ldm/audio/dit.py +++ b/comfy/ldm/audio/dit.py @@ -425,19 +425,16 @@ class Attention(nn.Module): if n == 1 and causal: causal = False - if h != kv_h: - # Repeat interleave kv_heads to match q_heads - heads_per_kv_head = h // kv_h - k, v = map(lambda t: t.repeat_interleave(heads_per_kv_head, dim = 1), (k, v)) + gqa_kwargs = {"enable_gqa": True} if h != kv_h else {} if self.differential: q, q_diff = q.unbind(dim=1) k, k_diff = k.unbind(dim=1) - out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options) - out_diff = optimized_attention(q_diff, k_diff, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options) + out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs) + out_diff = optimized_attention(q_diff, k_diff, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs) out = out - out_diff else: - out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options) + out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs) out = self.to_out(out) diff --git a/comfy/ldm/boogu/model.py b/comfy/ldm/boogu/model.py new file mode 100644 index 000000000..ca88bdeb1 --- /dev/null +++ b/comfy/ldm/boogu/model.py @@ -0,0 +1,318 @@ +# Boogu-Image-0.1 transformer +# Architecture is an OmniGen2 derivative (see comfy/ldm/omnigen/omnigen2.py) with an +# added dual-stream ("double_stream") stage before the single-stream layers, conditioned +# by a Qwen3-VL multimodal LLM. Reuses the OmniGen2/Lumina building blocks and the Flux +# RoPE core, the only new component is the double-stream block + the hybrid forward order. + +from typing import Optional, Tuple + +import torch +import torch.nn as nn +from einops import rearrange + +import comfy.ldm.common_dit +import comfy.ldm.omnigen.omnigen2 +from comfy.ldm.modules.attention import optimized_attention_masked +from comfy.ldm.omnigen.omnigen2 import ( + OmniGen2RotaryPosEmbed, + Lumina2CombinedTimestepCaptionEmbedding, + LuminaRMSNormZero, + LuminaLayerNormContinuous, + LuminaFeedForward, + Attention, + OmniGen2TransformerBlock, + apply_rotary_emb, +) + +class BooguDoubleStreamProcessor(nn.Module): + # Joint attention over [instruct ; img] with separate per-stream q/k/v and output projections. + def __init__(self, dim, head_dim, heads, kv_heads, dtype=None, device=None, operations=None): + super().__init__() + query_dim = head_dim * heads + kv_dim = head_dim * kv_heads + + self.img_to_q = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + self.img_to_k = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + self.img_to_v = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + + self.instruct_to_q = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + self.instruct_to_k = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + self.instruct_to_v = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + + self.instruct_out = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + self.img_out = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + + def forward(self, attn, img_hidden_states, instruct_hidden_states, rotary_emb, attention_mask=None, transformer_options={}): + batch_size = img_hidden_states.shape[0] + L_instruct = instruct_hidden_states.shape[1] + + img_q = self.img_to_q(img_hidden_states) + img_k = self.img_to_k(img_hidden_states) + img_v = self.img_to_v(img_hidden_states) + + instruct_q = self.instruct_to_q(instruct_hidden_states) + instruct_k = self.instruct_to_k(instruct_hidden_states) + instruct_v = self.instruct_to_v(instruct_hidden_states) + + # Concatenate instruction first, then image (matches reference processor order). + query = torch.cat([instruct_q, img_q], dim=1) + key = torch.cat([instruct_k, img_k], dim=1) + value = torch.cat([instruct_v, img_v], dim=1) + + query = query.view(batch_size, -1, attn.heads, attn.dim_head) + key = key.view(batch_size, -1, attn.kv_heads, attn.dim_head) + value = value.view(batch_size, -1, attn.kv_heads, attn.dim_head) + + query = attn.norm_q(query) + key = attn.norm_k(key) + + if rotary_emb is not None: + query = apply_rotary_emb(query, rotary_emb) + key = apply_rotary_emb(key, rotary_emb) + + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + gqa_kwargs = {"enable_gqa": True} if attn.kv_heads < attn.heads else {} + hidden_states = optimized_attention_masked(query, key, value, attn.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options, **gqa_kwargs) + + # Split back to instruction/image, apply per-stream output projections, recombine. + instruct_hidden_states = self.instruct_out(hidden_states[:, :L_instruct]) + img_hidden_states = self.img_out(hidden_states[:, L_instruct:]) + hidden_states = torch.cat([instruct_hidden_states, img_hidden_states], dim=1) + + hidden_states = attn.to_out[0](hidden_states) + return hidden_states + + +class BooguJointAttention(nn.Module): + # Holds the shared q/k RMSNorm + final output projection + def __init__(self, dim, head_dim, heads, kv_heads, eps=1e-5, dtype=None, device=None, operations=None): + super().__init__() + self.heads = heads + self.kv_heads = kv_heads + self.dim_head = head_dim + self.scale = head_dim ** -0.5 + + self.norm_q = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.norm_k = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.to_out = nn.Sequential( + operations.Linear(heads * head_dim, dim, bias=False, dtype=dtype, device=device), + nn.Dropout(0.0), + ) + self.processor = BooguDoubleStreamProcessor(dim, head_dim, heads, kv_heads, dtype=dtype, device=device, operations=operations) + + def forward(self, img_hidden_states, instruct_hidden_states, rotary_emb, attention_mask=None, transformer_options={}): + return self.processor(self, img_hidden_states, instruct_hidden_states, rotary_emb, attention_mask, transformer_options=transformer_options) + + +class BooguDoubleStreamBlock(nn.Module): + # Dual-stream block: joint attention over [instruct ; img] + image self-attention, each stream with its own modulation/MLP. + def __init__(self, dim, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, dtype=None, device=None, operations=None): + super().__init__() + head_dim = dim // num_attention_heads + + self.img_instruct_attn = BooguJointAttention(dim, head_dim, num_attention_heads, num_kv_heads, eps=1e-5, dtype=dtype, device=device, operations=operations) + self.img_self_attn = Attention( + query_dim=dim, dim_head=head_dim, heads=num_attention_heads, kv_heads=num_kv_heads, + eps=1e-5, bias=False, dtype=dtype, device=device, operations=operations, + ) + + self.img_feed_forward = LuminaFeedForward(dim=dim, inner_dim=4 * dim, multiple_of=multiple_of, dtype=dtype, device=device, operations=operations) + self.instruct_feed_forward = LuminaFeedForward(dim=dim, inner_dim=4 * dim, multiple_of=multiple_of, dtype=dtype, device=device, operations=operations) + + self.img_norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.img_norm2 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.img_norm3 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.instruct_norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.instruct_norm2 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + + self.img_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.img_self_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.img_ffn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.img_ffn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + + self.instruct_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.instruct_ffn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.instruct_ffn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + + def forward(self, img_hidden_states, instruct_hidden_states, joint_rotary_emb, img_rotary_emb, temb, joint_attention_mask=None, img_attention_mask=None, transformer_options={}): + L_instruct = instruct_hidden_states.shape[1] + + img_norm1_out, img_gate_msa, img_scale_mlp, img_gate_mlp = self.img_norm1(img_hidden_states, temb) + img_norm2_out, img_shift_mlp, _, _ = self.img_norm2(img_hidden_states, temb) + img_norm3_out, img_gate_self, _, _ = self.img_norm3(img_hidden_states, temb) + + instruct_norm1_out, instruct_gate_msa, instruct_scale_mlp, instruct_gate_mlp = self.instruct_norm1(instruct_hidden_states, temb) + instruct_norm2_out, instruct_shift_mlp, _, _ = self.instruct_norm2(instruct_hidden_states, temb) + + joint_attn_out = self.img_instruct_attn(img_norm1_out, instruct_norm1_out, joint_rotary_emb, joint_attention_mask, transformer_options=transformer_options) + instruct_attn_out = joint_attn_out[:, :L_instruct] + img_attn_out = joint_attn_out[:, L_instruct:] + + img_self_attn_out = self.img_self_attn(img_norm3_out, img_norm3_out, img_attention_mask, img_rotary_emb, transformer_options=transformer_options) + + img_hidden_states = img_hidden_states + img_gate_msa.unsqueeze(1).tanh() * self.img_attn_norm(img_attn_out) + img_hidden_states = img_hidden_states + img_gate_self.unsqueeze(1).tanh() * self.img_self_attn_norm(img_self_attn_out) + img_mlp_input = (1 + img_scale_mlp.unsqueeze(1)) * img_norm2_out + img_shift_mlp.unsqueeze(1) + img_mlp_out = self.img_feed_forward(self.img_ffn_norm1(img_mlp_input)) + img_hidden_states = img_hidden_states + img_gate_mlp.unsqueeze(1).tanh() * self.img_ffn_norm2(img_mlp_out) + + instruct_hidden_states = instruct_hidden_states + instruct_gate_msa.unsqueeze(1).tanh() * self.instruct_attn_norm(instruct_attn_out) + instruct_mlp_input = (1 + instruct_scale_mlp.unsqueeze(1)) * instruct_norm2_out + instruct_shift_mlp.unsqueeze(1) + instruct_mlp_out = self.instruct_feed_forward(self.instruct_ffn_norm1(instruct_mlp_input)) + instruct_hidden_states = instruct_hidden_states + instruct_gate_mlp.unsqueeze(1).tanh() * self.instruct_ffn_norm2(instruct_mlp_out) + + return img_hidden_states, instruct_hidden_states + + +class BooguTransformer2DModel(nn.Module): + def __init__( + self, + patch_size: int = 2, + in_channels: int = 16, + out_channels: Optional[int] = None, + hidden_size: int = 3360, + num_layers: int = 32, + num_double_stream_layers: int = 8, + num_refiner_layers: int = 2, + num_attention_heads: int = 28, + num_kv_heads: int = 7, + multiple_of: int = 256, + ffn_dim_multiplier: Optional[float] = None, + norm_eps: float = 1e-5, + axes_dim_rope: Tuple[int, int, int] = (40, 40, 40), + axes_lens: Tuple[int, int, int] = (2048, 1664, 1664), + instruction_feat_dim: int = 4096, + timestep_scale: float = 1000.0, + image_model=None, + device=None, dtype=None, operations=None, + ): + super().__init__() + + self.patch_size = patch_size + self.out_channels = out_channels or in_channels + self.hidden_size = hidden_size + self.dtype = dtype + + self.rope_embedder = OmniGen2RotaryPosEmbed( + theta=10000, + axes_dim=axes_dim_rope, + axes_lens=axes_lens, + patch_size=patch_size, + ) + + self.x_embedder = operations.Linear(patch_size * patch_size * in_channels, hidden_size, dtype=dtype, device=device) + self.ref_image_patch_embedder = operations.Linear(patch_size * patch_size * in_channels, hidden_size, dtype=dtype, device=device) + + self.time_caption_embed = Lumina2CombinedTimestepCaptionEmbedding( + hidden_size=hidden_size, + text_feat_dim=instruction_feat_dim, + norm_eps=norm_eps, + timestep_scale=timestep_scale, dtype=dtype, device=device, operations=operations + ) + + self.noise_refiner = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=True, dtype=dtype, device=device, operations=operations) + for _ in range(num_refiner_layers) + ]) + + self.ref_image_refiner = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=True, dtype=dtype, device=device, operations=operations) + for _ in range(num_refiner_layers) + ]) + + self.context_refiner = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=False, dtype=dtype, device=device, operations=operations) + for _ in range(num_refiner_layers) + ]) + + self.double_stream_layers = nn.ModuleList([ + BooguDoubleStreamBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, dtype=dtype, device=device, operations=operations) + for _ in range(num_double_stream_layers) + ]) + + self.single_stream_layers = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=True, dtype=dtype, device=device, operations=operations) + for _ in range(num_layers) + ]) + + self.norm_out = LuminaLayerNormContinuous( + embedding_dim=hidden_size, + conditioning_embedding_dim=min(hidden_size, 1024), + elementwise_affine=False, + eps=1e-6, + out_dim=patch_size * patch_size * self.out_channels, dtype=dtype, device=device, operations=operations + ) + + self.image_index_embedding = nn.Parameter(torch.empty(5, hidden_size, device=device, dtype=dtype)) + + # Patchify/refine helpers are identical to OmniGen2; reuse via bound methods. + flat_and_pad_to_seq = comfy.ldm.omnigen.omnigen2.OmniGen2Transformer2DModel.flat_and_pad_to_seq + img_patch_embed_and_refine = comfy.ldm.omnigen.omnigen2.OmniGen2Transformer2DModel.img_patch_embed_and_refine + + def forward(self, x, timesteps, context, num_tokens, ref_latents=None, attention_mask=None, transformer_options={}, **kwargs): + B, C, H, W = x.shape + hidden_states = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size)) + _, _, H_padded, W_padded = hidden_states.shape + timestep = 1.0 - timesteps + text_hidden_states = context + text_attention_mask = attention_mask + ref_image_hidden_states = ref_latents + device = hidden_states.device + + temb, text_hidden_states = self.time_caption_embed(timestep, text_hidden_states, hidden_states[0].dtype) + + ( + hidden_states, ref_image_hidden_states, + img_mask, ref_img_mask, + l_effective_ref_img_len, l_effective_img_len, + ref_img_sizes, img_sizes, + ) = self.flat_and_pad_to_seq(hidden_states, ref_image_hidden_states) + + ( + context_rotary_emb, ref_img_rotary_emb, noise_rotary_emb, + rotary_emb, encoder_seq_lengths, seq_lengths, + ) = self.rope_embedder( + hidden_states.shape[0], text_hidden_states.shape[1], [num_tokens] * text_hidden_states.shape[0], + l_effective_ref_img_len, l_effective_img_len, + ref_img_sizes, img_sizes, device, + ) + + for layer in self.context_refiner: + text_hidden_states = layer(text_hidden_states, text_attention_mask, context_rotary_emb, transformer_options=transformer_options) + + img_len = hidden_states.shape[1] + combined_img_hidden_states = self.img_patch_embed_and_refine( + hidden_states, ref_image_hidden_states, + img_mask, ref_img_mask, + noise_rotary_emb, ref_img_rotary_emb, + l_effective_ref_img_len, l_effective_img_len, + temb, + transformer_options=transformer_options, + ) + + # Double-stream stage: the image self-attention only sees the [ref ; noise] tokens, + # which sit after the instruction tokens in the joint rope. + L_instruct = text_hidden_states.shape[1] + combined_img_rotary_emb = rotary_emb[:, L_instruct:] + for layer in self.double_stream_layers: + combined_img_hidden_states, text_hidden_states = layer( + combined_img_hidden_states, text_hidden_states, + rotary_emb, combined_img_rotary_emb, temb, + joint_attention_mask=None, img_attention_mask=None, + transformer_options=transformer_options, + ) + + hidden_states = torch.cat([text_hidden_states, combined_img_hidden_states], dim=1) + + for layer in self.single_stream_layers: + hidden_states = layer(hidden_states, None, rotary_emb, temb, transformer_options=transformer_options) + + hidden_states = self.norm_out(hidden_states, temb) + + p = self.patch_size + output = rearrange(hidden_states[:, -img_len:], 'b (h w) (p1 p2 c) -> b c (h p1) (w p2)', h=H_padded // p, w=W_padded // p, p1=p, p2=p)[:, :, :H, :W] + + return -output diff --git a/comfy/ldm/colormap.py b/comfy/ldm/colormap.py new file mode 100644 index 000000000..1f4d88bd9 --- /dev/null +++ b/comfy/ldm/colormap.py @@ -0,0 +1,25 @@ +"""Colormap utilities for depth and geometry visualisation.""" + +from __future__ import annotations + +import torch + + +def turbo(x: torch.Tensor) -> torch.Tensor: + """Anton Mikhailov polynomial approximation of the Turbo colormap. + + Args: + x: Float tensor with values in [0, 1]. + + Returns: + RGB tensor of the same shape as ``x`` with a trailing size-3 dimension. + """ + x = x.clamp(0.0, 1.0) + x2 = x * x + x3 = x2 * x + x4 = x2 * x2 + x5 = x4 * x + r = 0.13572138 + 4.61539260*x - 42.66032258*x2 + 132.13108234*x3 - 152.94239396*x4 + 59.28637943*x5 + g = 0.09140261 + 2.19418839*x + 4.84296658*x2 - 14.18503333*x3 + 4.27729857*x4 + 2.82956604*x5 + b = 0.10667330 + 12.64194608*x - 60.58204836*x2 + 110.36276771*x3 - 89.90310912*x4 + 27.34824973*x5 + return torch.stack([r, g, b], dim=-1).clamp(0.0, 1.0) diff --git a/comfy/ldm/cosmos/predict2.py b/comfy/ldm/cosmos/predict2.py index 671fe834d..aec874815 100644 --- a/comfy/ldm/cosmos/predict2.py +++ b/comfy/ldm/cosmos/predict2.py @@ -515,7 +515,7 @@ class Block(nn.Module): h=H, w=W, ) - x_B_T_H_W_D = x_B_T_H_W_D + gate_self_attn_B_T_1_1_D.to(residual_dtype) * result_B_T_H_W_D.to(residual_dtype) + x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_self_attn_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype)) def _x_fn( _x_B_T_H_W_D: torch.Tensor, @@ -548,7 +548,7 @@ class Block(nn.Module): shift_cross_attn_B_T_1_1_D, transformer_options=transformer_options, ) - x_B_T_H_W_D = result_B_T_H_W_D.to(residual_dtype) * gate_cross_attn_B_T_1_1_D.to(residual_dtype) + x_B_T_H_W_D + x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_cross_attn_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype)) normalized_x_B_T_H_W_D = _fn( x_B_T_H_W_D, @@ -557,7 +557,7 @@ class Block(nn.Module): shift_mlp_B_T_1_1_D, ) result_B_T_H_W_D = self.mlp(normalized_x_B_T_H_W_D.to(compute_dtype)) - x_B_T_H_W_D = x_B_T_H_W_D + gate_mlp_B_T_1_1_D.to(residual_dtype) * result_B_T_H_W_D.to(residual_dtype) + x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_mlp_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype)) return x_B_T_H_W_D diff --git a/comfy/ldm/depth_anything_3/camera.py b/comfy/ldm/depth_anything_3/camera.py new file mode 100644 index 000000000..65a57d66f --- /dev/null +++ b/comfy/ldm/depth_anything_3/camera.py @@ -0,0 +1,177 @@ +"""Camera-token encoder and decoder for Depth Anything 3.""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from comfy.ldm.modules.attention import optimized_attention_for_device +from .transform import affine_inverse, extri_intri_to_pose_encoding + + +# ----------------------------------------------------------------------- +# Building blocks (mirror depth_anything_3.model.utils.{attention,block}) +# ----------------------------------------------------------------------- + + +class _Mlp(nn.Module): + """Standard 2-layer MLP with GELU. Matches upstream ``utils.attention.Mlp``.""" + + def __init__(self, in_features, hidden_features=None, out_features=None, *, device=None, dtype=None, operations=None): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = operations.Linear(in_features, hidden_features, bias=True, device=device, dtype=dtype) + self.fc2 = operations.Linear(hidden_features, out_features, bias=True, device=device, dtype=dtype) + + def forward(self, x): + return self.fc2(F.gelu(self.fc1(x))) + + +class _LayerScale(nn.Module): + """Per-channel learnable scaling. Matches upstream LayerScale.""" + + def __init__(self, dim, *, device=None, dtype=None): + super().__init__() + self.gamma = nn.Parameter(torch.empty(dim, device=device, dtype=dtype)) + + def forward(self, x): + return x * self.gamma.to(dtype=x.dtype, device=x.device) + + +class _Attention(nn.Module): + """ Self-attention with fused QKV projection. Mirrors upstream utils.attention.Attention; + Layout matches the HF safetensors (attn.qkv.{weight,bias} and attn.proj.{weight,bias}).""" + + def __init__(self, dim, num_heads, *, device=None, dtype=None, operations=None): + super().__init__() + assert dim % num_heads == 0 + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.qkv = operations.Linear(dim, dim * 3, bias=True, device=device, dtype=dtype) + self.proj = operations.Linear(dim, dim, bias=True, device=device, dtype=dtype) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, C) + q, k, v = qkv.unbind(2) # each (B, N, C) + attn_fn = optimized_attention_for_device(x.device, small_input=True) + out = attn_fn(q, k, v, heads=self.num_heads) + return self.proj(out) + + +class _Block(nn.Module): + """Pre-norm transformer block with LayerScale. Used by :class:CameraEnc. Layout follows upstream utils.block.Block.""" + + def __init__(self, dim, num_heads, mlp_ratio=4, init_values=0.01, *, device=None, dtype=None, operations=None): + super().__init__() + self.norm1 = operations.LayerNorm(dim, device=device, dtype=dtype) + self.attn = _Attention(dim, num_heads, device=device, dtype=dtype, operations=operations) + self.ls1 = _LayerScale(dim, device=device, dtype=dtype) if init_values else nn.Identity() + self.norm2 = operations.LayerNorm(dim, device=device, dtype=dtype) + self.mlp = _Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), device=device, dtype=dtype, operations=operations) + self.ls2 = _LayerScale(dim, device=device, dtype=dtype) if init_values else nn.Identity() + + def forward(self, x): + x = x + self.ls1(self.attn(self.norm1(x))) + x = x + self.ls2(self.mlp(self.norm2(x))) + return x + + +class CameraEnc(nn.Module): + """Encode per-view (extrinsics, intrinsics) into a camera token. + + Maps a 9-D pose-encoding vector through a small MLP up to the backbone's + ``embed_dim``, then runs ``trunk_depth`` transformer blocks. The output + has shape ``(B, S, embed_dim)`` and is injected at block ``alt_start`` + of the DINOv2 backbone in place of the cls token. + + Parameters mirror the upstream ``cam_enc.py`` so HF weights load directly. + """ + + def __init__( + self, + dim_out: int = 1024, + dim_in: int = 9, + trunk_depth: int = 4, + target_dim: int = 9, + num_heads: int = 16, + mlp_ratio: int = 4, + init_values: float = 0.01, + *, + device=None, dtype=None, operations=None, + **_kwargs, + ): + super().__init__() + self.target_dim = target_dim + self.trunk_depth = trunk_depth + self.trunk = nn.Sequential(*[ + _Block(dim_out, num_heads=num_heads, mlp_ratio=mlp_ratio, + init_values=init_values, + device=device, dtype=dtype, operations=operations) + for _ in range(trunk_depth) + ]) + self.token_norm = operations.LayerNorm(dim_out, device=device, dtype=dtype) + self.trunk_norm = operations.LayerNorm(dim_out, device=device, dtype=dtype) + self.pose_branch = _Mlp( + in_features=dim_in, + hidden_features=dim_out // 2, + out_features=dim_out, + device=device, dtype=dtype, operations=operations, + ) + + def forward(self, extrinsics: torch.Tensor, intrinsics: torch.Tensor, + image_size_hw) -> torch.Tensor: + """Encode camera parameters into ``(B, S, dim_out)`` tokens.""" + c2ws = affine_inverse(extrinsics) + pose_encoding = extri_intri_to_pose_encoding(c2ws, intrinsics, image_size_hw) + tokens = self.pose_branch(pose_encoding.to(self.pose_branch.fc1.weight.dtype)) + tokens = self.token_norm(tokens) + tokens = self.trunk(tokens) + tokens = self.trunk_norm(tokens) + return tokens + + +class CameraDec(nn.Module): + """Decode the final cam token into a 9-D pose encoding. + + Output layout: ``[T(3), quat_xyzw(4), fov_h, fov_w]``. The translation is + always predicted by the network; the quaternion and FoV can either be + predicted or supplied via ``camera_encoding`` (used at training time + when GT cameras are available -- not exercised at inference here). + + Parameters mirror the upstream ``cam_dec.py`` so HF weights load directly. + """ + + def __init__(self, dim_in: int = 1536, + *, device=None, dtype=None, operations=None, **_kwargs): + super().__init__() + d = dim_in + self.backbone = nn.Sequential( + operations.Linear(d, d, device=device, dtype=dtype), + nn.ReLU(), + operations.Linear(d, d, device=device, dtype=dtype), + nn.ReLU(), + ) + self.fc_t = operations.Linear(d, 3, device=device, dtype=dtype) + self.fc_qvec = operations.Linear(d, 4, device=device, dtype=dtype) + self.fc_fov = nn.Sequential( + operations.Linear(d, 2, device=device, dtype=dtype), + nn.ReLU(), + ) + + def forward(self, feat: torch.Tensor, + camera_encoding: "torch.Tensor | None" = None) -> torch.Tensor: + """Decode ``(B, N, dim_in)`` cam tokens into ``(B, N, 9)`` pose enc.""" + B, N = feat.shape[:2] + feat = feat.reshape(B * N, -1) + feat = self.backbone(feat) + out_t = self.fc_t(feat.float()).reshape(B, N, 3) + if camera_encoding is None: + out_qvec = self.fc_qvec(feat.float()).reshape(B, N, 4) + out_fov = self.fc_fov(feat.float()).reshape(B, N, 2) + else: + out_qvec = camera_encoding[..., 3:7] + out_fov = camera_encoding[..., -2:] + return torch.cat([out_t, out_qvec, out_fov], dim=-1) diff --git a/comfy/ldm/depth_anything_3/dpt.py b/comfy/ldm/depth_anything_3/dpt.py new file mode 100644 index 000000000..fb940873b --- /dev/null +++ b/comfy/ldm/depth_anything_3/dpt.py @@ -0,0 +1,489 @@ +"""DPT / DualDPT heads for Depth Anything 3.""" + +from __future__ import annotations + +from typing import List, Optional, Sequence, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Permute(nn.Module): + def __init__(self, dims: Tuple[int, ...]): + super().__init__() + self.dims = dims + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x.permute(*self.dims) + + +def _custom_interpolate( + x: torch.Tensor, + size: Optional[Tuple[int, int]] = None, + scale_factor: Optional[float] = None, + mode: str = "bilinear", + align_corners: bool = True, +) -> torch.Tensor: + if size is None: + assert scale_factor is not None + size = (int(x.shape[-2] * scale_factor), int(x.shape[-1] * scale_factor)) + INT_MAX = 1610612736 + total = size[0] * size[1] * x.shape[0] * x.shape[1] + if total > INT_MAX: + chunks = torch.chunk(x, chunks=(total // INT_MAX) + 1, dim=0) + outs = [F.interpolate(c, size=size, mode=mode, align_corners=align_corners) for c in chunks] + return torch.cat(outs, dim=0).contiguous() + return F.interpolate(x, size=size, mode=mode, align_corners=align_corners) + + +def _create_uv_grid(width: int, height: int, aspect_ratio: float, dtype, device) -> torch.Tensor: + """Normalised UV grid spanning (-x_span, -y_span)..(x_span, y_span).""" + diag_factor = (aspect_ratio ** 2 + 1.0) ** 0.5 + span_x = aspect_ratio / diag_factor + span_y = 1.0 / diag_factor + left_x = -span_x * (width - 1) / width + right_x = span_x * (width - 1) / width + top_y = -span_y * (height - 1) / height + bottom_y = span_y * (height - 1) / height + x_coords = torch.linspace(left_x, right_x, steps=width, dtype=dtype, device=device) + y_coords = torch.linspace(top_y, bottom_y, steps=height, dtype=dtype, device=device) + uu, vv = torch.meshgrid(x_coords, y_coords, indexing="xy") + return torch.stack((uu, vv), dim=-1) # (H, W, 2) + + +def _make_sincos_pos_embed(embed_dim: int, pos: torch.Tensor, omega_0: float = 100.0) -> torch.Tensor: + omega = torch.arange(embed_dim // 2, dtype=torch.float32, device=pos.device) + omega = 1.0 / omega_0 ** (omega / (embed_dim / 2.0)) + pos = pos.reshape(-1) + out = torch.einsum("m,d->md", pos, omega) + return torch.cat([out.sin(), out.cos()], dim=1).float() + + +def _position_grid_to_embed(pos_grid: torch.Tensor, embed_dim: int, omega_0: float = 100.0) -> torch.Tensor: + H, W, _ = pos_grid.shape + pos_flat = pos_grid.reshape(-1, 2) + emb_x = _make_sincos_pos_embed(embed_dim // 2, pos_flat[:, 0], omega_0=omega_0) + emb_y = _make_sincos_pos_embed(embed_dim // 2, pos_flat[:, 1], omega_0=omega_0) + emb = torch.cat([emb_x, emb_y], dim=-1) + return emb.view(H, W, embed_dim) + + +def _add_pos_embed(x: torch.Tensor, W: int, H: int, ratio: float = 0.1) -> torch.Tensor: + """Stateless UV positional embedding added to a feature map (B, C, h, w).""" + pw, ph = x.shape[-1], x.shape[-2] + pe = _create_uv_grid(pw, ph, aspect_ratio=W / H, dtype=x.dtype, device=x.device) + pe = _position_grid_to_embed(pe, x.shape[1]) * ratio + pe = pe.permute(2, 0, 1)[None].expand(x.shape[0], -1, -1, -1).to(dtype=x.dtype) + return x + pe + + +def _apply_activation(x: torch.Tensor, activation: str) -> torch.Tensor: + act = (activation or "linear").lower() + if act == "exp": + return torch.exp(x) + if act == "expp1": + return torch.exp(x) + 1 + if act == "expm1": + return torch.expm1(x) + if act == "relu": + return torch.relu(x) + if act == "sigmoid": + return torch.sigmoid(x) + if act == "softplus": + return F.softplus(x) + if act == "tanh": + return torch.tanh(x) + return x + + +# ----------------------------------------------------------------------------- +# Fusion building blocks +# ----------------------------------------------------------------------------- + + +class ResidualConvUnit(nn.Module): + def __init__(self, features: int, device=None, dtype=None, operations=None): + super().__init__() + self.conv1 = operations.Conv2d(features, features, 3, 1, 1, bias=True, device=device, dtype=dtype) + self.conv2 = operations.Conv2d(features, features, 3, 1, 1, bias=True, device=device, dtype=dtype) + self.activation = nn.ReLU(inplace=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = self.activation(x) + out = self.conv1(out) + out = self.activation(out) + out = self.conv2(out) + return out + x + + +class FeatureFusionBlock(nn.Module): + def __init__(self, features: int, has_residual: bool = True, align_corners: bool = True, device=None, dtype=None, operations=None): + super().__init__() + self.align_corners = align_corners + self.has_residual = has_residual + if has_residual: + self.resConfUnit1 = ResidualConvUnit(features, device=device, dtype=dtype, operations=operations) + else: + self.resConfUnit1 = None + self.resConfUnit2 = ResidualConvUnit(features, device=device, dtype=dtype, operations=operations) + self.out_conv = operations.Conv2d(features, features, 1, 1, 0, bias=True, device=device, dtype=dtype) + + def forward(self, *xs: torch.Tensor, size: Optional[Tuple[int, int]] = None) -> torch.Tensor: + y = xs[0] + if self.has_residual and len(xs) > 1 and self.resConfUnit1 is not None: + y = y + self.resConfUnit1(xs[1]) + y = self.resConfUnit2(y) + if size is None: + up_kwargs = {"scale_factor": 2.0} + else: + up_kwargs = {"size": size} + y = _custom_interpolate(y, **up_kwargs, mode="bilinear", align_corners=self.align_corners) + y = self.out_conv(y) + return y + + +class _Scratch(nn.Module): + """Container that mirrors upstream ``scratch`` attribute layout.""" + + +def _make_scratch(in_shape: List[int], out_shape: int, device=None, dtype=None, operations=None) -> _Scratch: + scratch = _Scratch() + scratch.layer1_rn = operations.Conv2d(in_shape[0], out_shape, 3, 1, 1, bias=False, device=device, dtype=dtype) + scratch.layer2_rn = operations.Conv2d(in_shape[1], out_shape, 3, 1, 1, bias=False, device=device, dtype=dtype) + scratch.layer3_rn = operations.Conv2d(in_shape[2], out_shape, 3, 1, 1, bias=False, device=device, dtype=dtype) + scratch.layer4_rn = operations.Conv2d(in_shape[3], out_shape, 3, 1, 1, bias=False, device=device, dtype=dtype) + return scratch + + +def _make_fusion_block(features: int, has_residual: bool = True, device=None, dtype=None, operations=None) -> FeatureFusionBlock: + return FeatureFusionBlock(features, has_residual=has_residual, align_corners=True, device=device, dtype=dtype, operations=operations) + + +# ----------------------------------------------------------------------------- +# DPT (single head + optional sky head) -- used by DA3Mono/Metric +# ----------------------------------------------------------------------------- + + +class DPT(nn.Module): + """Single-head DPT used by DA3Mono-Large and DA3Metric-Large.""" + + def __init__( + self, + dim_in: int, + patch_size: int = 14, + output_dim: int = 1, + activation: str = "exp", + conf_activation: str = "expp1", + features: int = 256, + out_channels: Sequence[int] = (256, 512, 1024, 1024), + pos_embed: bool = False, + down_ratio: int = 1, + head_name: str = "depth", + use_sky_head: bool = True, + sky_name: str = "sky", + sky_activation: str = "relu", + norm_type: str = "idt", + device=None, dtype=None, operations=None, + ): + super().__init__() + self.patch_size = patch_size + self.activation = activation + self.conf_activation = conf_activation + self.pos_embed = pos_embed + self.down_ratio = down_ratio + self.head_main = head_name + self.sky_name = sky_name + self.out_dim = output_dim + self.has_conf = output_dim > 1 + self.use_sky_head = use_sky_head + self.sky_activation = sky_activation + self.intermediate_layer_idx: Tuple[int, int, int, int] = (0, 1, 2, 3) + + if norm_type == "layer": + self.norm = operations.LayerNorm(dim_in, device=device, dtype=dtype) + else: + self.norm = nn.Identity() + + out_channels = list(out_channels) + self.projects = nn.ModuleList([ + operations.Conv2d(dim_in, oc, kernel_size=1, stride=1, padding=0, device=device, dtype=dtype) + for oc in out_channels + ]) + self.resize_layers = nn.ModuleList([ + operations.ConvTranspose2d(out_channels[0], out_channels[0], kernel_size=4, stride=4, padding=0, device=device, dtype=dtype), + operations.ConvTranspose2d(out_channels[1], out_channels[1], kernel_size=2, stride=2, padding=0, device=device, dtype=dtype), + nn.Identity(), + operations.Conv2d(out_channels[3], out_channels[3], kernel_size=3, stride=2, padding=1, device=device, dtype=dtype), + ]) + + self.scratch = _make_scratch(out_channels, features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet1 = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet2 = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet3 = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet4 = _make_fusion_block(features, has_residual=False, device=device, dtype=dtype, operations=operations) + + head_features_1 = features + head_features_2 = 32 + self.scratch.output_conv1 = operations.Conv2d( + head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1, + device=device, dtype=dtype, + ) + self.scratch.output_conv2 = nn.Sequential( + operations.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1, device=device, dtype=dtype), + nn.ReLU(inplace=False), + operations.Conv2d(head_features_2, output_dim, kernel_size=1, stride=1, padding=0, device=device, dtype=dtype), + ) + + if self.use_sky_head: + self.scratch.sky_output_conv2 = nn.Sequential( + operations.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1, device=device, dtype=dtype), + nn.ReLU(inplace=False), + operations.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0, device=device, dtype=dtype), + ) + + def forward(self, feats: List[torch.Tensor], H: int, W: int, patch_start_idx: int = 0, **_kwargs) -> dict: + # feats[i][0] is the patch-token tensor with shape (B, S, N_patch, C) + B, S, N, C = feats[0][0].shape + feats_flat = [feat[0].reshape(B * S, N, C) for feat in feats] + + ph, pw = H // self.patch_size, W // self.patch_size + resized = [] + for stage_idx, take_idx in enumerate(self.intermediate_layer_idx): + x = feats_flat[take_idx][:, patch_start_idx:] + x = self.norm(x) + x = x.permute(0, 2, 1).contiguous().reshape(B * S, C, ph, pw) + x = self.projects[stage_idx](x) + if self.pos_embed: + x = _add_pos_embed(x, W, H) + x = self.resize_layers[stage_idx](x) + resized.append(x) + + l1_rn = self.scratch.layer1_rn(resized[0]) + l2_rn = self.scratch.layer2_rn(resized[1]) + l3_rn = self.scratch.layer3_rn(resized[2]) + l4_rn = self.scratch.layer4_rn(resized[3]) + + out = self.scratch.refinenet4(l4_rn, size=l3_rn.shape[2:]) + out = self.scratch.refinenet3(out, l3_rn, size=l2_rn.shape[2:]) + out = self.scratch.refinenet2(out, l2_rn, size=l1_rn.shape[2:]) + out = self.scratch.refinenet1(out, l1_rn) + + h_out = int(ph * self.patch_size / self.down_ratio) + w_out = int(pw * self.patch_size / self.down_ratio) + + fused = self.scratch.output_conv1(out) + fused = _custom_interpolate(fused, (h_out, w_out), mode="bilinear", align_corners=True) + if self.pos_embed: + fused = _add_pos_embed(fused, W, H) + feat = fused + + main_logits = self.scratch.output_conv2(feat) + outs = {} + if self.has_conf: + fmap = main_logits.permute(0, 2, 3, 1) + pred = _apply_activation(fmap[..., :-1], self.activation) + conf = _apply_activation(fmap[..., -1], self.conf_activation) + outs[self.head_main] = pred.squeeze(-1).view(B, S, *pred.shape[1:-1]) + outs[f"{self.head_main}_conf"] = conf.view(B, S, *conf.shape[1:]) + else: + pred = _apply_activation(main_logits, self.activation) + outs[self.head_main] = pred.squeeze(1).view(B, S, *pred.shape[2:]) + + if self.use_sky_head: + sky_logits = self.scratch.sky_output_conv2(feat) + if self.sky_activation.lower() == "sigmoid": + sky = torch.sigmoid(sky_logits) + elif self.sky_activation.lower() == "relu": + sky = F.relu(sky_logits) + else: + sky = sky_logits + outs[self.sky_name] = sky.squeeze(1).view(B, S, *sky.shape[2:]) + + return outs + + +# ----------------------------------------------------------------------------- +# DualDPT (depth + auxiliary "ray" head) -- used by DA3-Small / DA3-Base +# ----------------------------------------------------------------------------- + + +class DualDPT(nn.Module): + """Two-head DPT used by DA3-Small / DA3-Base.""" + + def __init__( + self, + dim_in: int, + patch_size: int = 14, + output_dim: int = 2, + activation: str = "exp", + conf_activation: str = "expp1", + features: int = 256, + out_channels: Sequence[int] = (256, 512, 1024, 1024), + pos_embed: bool = True, + down_ratio: int = 1, + aux_pyramid_levels: int = 4, + aux_out1_conv_num: int = 5, + head_names: Tuple[str, str] = ("depth", "ray"), + device=None, dtype=None, operations=None, + ): + super().__init__() + self.patch_size = patch_size + self.activation = activation + self.conf_activation = conf_activation + self.pos_embed = pos_embed + self.down_ratio = down_ratio + self.aux_levels = aux_pyramid_levels + self.aux_out1_conv_num = aux_out1_conv_num + self.head_main, self.head_aux = head_names + self.intermediate_layer_idx: Tuple[int, int, int, int] = (0, 1, 2, 3) + # Toggle the auxiliary ray branch at runtime. Default off (mono path). + # DepthAnything3Net flips this on when running multi-view + ray-pose. + self.enable_aux: bool = False + + self.norm = operations.LayerNorm(dim_in, device=device, dtype=dtype) + out_channels = list(out_channels) + self.projects = nn.ModuleList([ + operations.Conv2d(dim_in, oc, kernel_size=1, stride=1, padding=0, device=device, dtype=dtype) + for oc in out_channels + ]) + self.resize_layers = nn.ModuleList([ + operations.ConvTranspose2d(out_channels[0], out_channels[0], kernel_size=4, stride=4, padding=0, device=device, dtype=dtype), + operations.ConvTranspose2d(out_channels[1], out_channels[1], kernel_size=2, stride=2, padding=0, device=device, dtype=dtype), + nn.Identity(), + operations.Conv2d(out_channels[3], out_channels[3], kernel_size=3, stride=2, padding=1, device=device, dtype=dtype), + ]) + + self.scratch = _make_scratch(out_channels, features, device=device, dtype=dtype, operations=operations) + # Main fusion chain + self.scratch.refinenet1 = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet2 = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet3 = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet4 = _make_fusion_block(features, has_residual=False, device=device, dtype=dtype, operations=operations) + # Auxiliary fusion chain (separate copies) + self.scratch.refinenet1_aux = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet2_aux = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet3_aux = _make_fusion_block(features, device=device, dtype=dtype, operations=operations) + self.scratch.refinenet4_aux = _make_fusion_block(features, has_residual=False, device=device, dtype=dtype, operations=operations) + + head_features_1 = features + head_features_2 = 32 + + # Main head neck + final projection + self.scratch.output_conv1 = operations.Conv2d( + head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1, + device=device, dtype=dtype, + ) + self.scratch.output_conv2 = nn.Sequential( + operations.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1, device=device, dtype=dtype), + nn.ReLU(inplace=False), + operations.Conv2d(head_features_2, output_dim, kernel_size=1, stride=1, padding=0, device=device, dtype=dtype), + ) + + # Aux pre-head per level (multi-level pyramid) + self.scratch.output_conv1_aux = nn.ModuleList([ + self._make_aux_out1_block(head_features_1, device=device, dtype=dtype, operations=operations) + for _ in range(self.aux_levels) + ]) + + # Aux final projection per level (includes LayerNorm permute path). + ln_seq = [Permute((0, 2, 3, 1)), + operations.LayerNorm(head_features_2, device=device, dtype=dtype), + Permute((0, 3, 1, 2))] + self.scratch.output_conv2_aux = nn.ModuleList([ + nn.Sequential( + operations.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1, device=device, dtype=dtype), + *ln_seq, + nn.ReLU(inplace=False), + operations.Conv2d(head_features_2, 7, kernel_size=1, stride=1, padding=0, device=device, dtype=dtype), + ) + for _ in range(self.aux_levels) + ]) + + @staticmethod + def _make_aux_out1_block(in_ch: int, *, device=None, dtype=None, operations=None) -> nn.Sequential: + # aux_out1_conv_num=5 in all Apache-2.0 variants. + return nn.Sequential( + operations.Conv2d(in_ch, in_ch // 2, 3, 1, 1, device=device, dtype=dtype), + operations.Conv2d(in_ch // 2, in_ch, 3, 1, 1, device=device, dtype=dtype), + operations.Conv2d(in_ch, in_ch // 2, 3, 1, 1, device=device, dtype=dtype), + operations.Conv2d(in_ch // 2, in_ch, 3, 1, 1, device=device, dtype=dtype), + operations.Conv2d(in_ch, in_ch // 2, 3, 1, 1, device=device, dtype=dtype), + ) + + def forward(self, feats: List[torch.Tensor], H: int, W: int, patch_start_idx: int = 0, **_kwargs) -> dict: + B, S, N, C = feats[0][0].shape + feats_flat = [feat[0].reshape(B * S, N, C) for feat in feats] + + ph, pw = H // self.patch_size, W // self.patch_size + resized = [] + for stage_idx, take_idx in enumerate(self.intermediate_layer_idx): + x = feats_flat[take_idx][:, patch_start_idx:] + x = self.norm(x) + x = x.permute(0, 2, 1).contiguous().reshape(B * S, C, ph, pw) + x = self.projects[stage_idx](x) + if self.pos_embed: + x = _add_pos_embed(x, W, H) + x = self.resize_layers[stage_idx](x) + resized.append(x) + + l1_rn = self.scratch.layer1_rn(resized[0]) + l2_rn = self.scratch.layer2_rn(resized[1]) + l3_rn = self.scratch.layer3_rn(resized[2]) + l4_rn = self.scratch.layer4_rn(resized[3]) + + # Main pyramid (output_conv1 is applied inside the upstream `_fuse`, + # before interpolation -- replicate that order here). + m = self.scratch.refinenet4(l4_rn, size=l3_rn.shape[2:]) + if self.enable_aux: + a4 = self.scratch.refinenet4_aux(l4_rn, size=l3_rn.shape[2:]) + aux_pyr = [a4] + m = self.scratch.refinenet3(m, l3_rn, size=l2_rn.shape[2:]) + if self.enable_aux: + aux_pyr.append(self.scratch.refinenet3_aux(aux_pyr[-1], l3_rn, size=l2_rn.shape[2:])) + m = self.scratch.refinenet2(m, l2_rn, size=l1_rn.shape[2:]) + if self.enable_aux: + aux_pyr.append(self.scratch.refinenet2_aux(aux_pyr[-1], l2_rn, size=l1_rn.shape[2:])) + m = self.scratch.refinenet1(m, l1_rn) + if self.enable_aux: + aux_pyr.append(self.scratch.refinenet1_aux(aux_pyr[-1], l1_rn)) + m = self.scratch.output_conv1(m) + + h_out = int(ph * self.patch_size / self.down_ratio) + w_out = int(pw * self.patch_size / self.down_ratio) + + m = _custom_interpolate(m, (h_out, w_out), mode="bilinear", align_corners=True) + if self.pos_embed: + m = _add_pos_embed(m, W, H) + main_logits = self.scratch.output_conv2(m) + fmap = main_logits.permute(0, 2, 3, 1) + depth_pred = _apply_activation(fmap[..., :-1], self.activation) + depth_conf = _apply_activation(fmap[..., -1], self.conf_activation) + + outs = { + self.head_main: depth_pred.squeeze(-1).view(B, S, *depth_pred.shape[1:-1]), + f"{self.head_main}_conf": depth_conf.view(B, S, *depth_conf.shape[1:]), + } + + if self.enable_aux: + # Auxiliary "ray" head (multi-level inside) -- only the last level + # is returned. Mirrors upstream ``DualDPT._fuse`` + ``_forward_impl``: + # each aux pyramid level goes through ``output_conv1_aux[i]`` + # (5-layer conv stack that ends at ``features // 2`` channels), + # then the last level optionally gets a pos-embed and finally + # ``output_conv2_aux[-1]``. + aux_processed = [ + self.scratch.output_conv1_aux[i](a) for i, a in enumerate(aux_pyr) + ] + last_aux = aux_processed[-1] + if self.pos_embed: + last_aux = _add_pos_embed(last_aux, W, H) + last_aux_logits = self.scratch.output_conv2_aux[-1](last_aux) + fmap_last = last_aux_logits.permute(0, 2, 3, 1) + # Channels: [ray(6), ray_conf(1)]; ray uses 'linear' activation. + aux_pred = fmap_last[..., :-1] + aux_conf = _apply_activation(fmap_last[..., -1], self.conf_activation) + outs[self.head_aux] = aux_pred.view(B, S, *aux_pred.shape[1:]) + outs[f"{self.head_aux}_conf"] = aux_conf.view(B, S, *aux_conf.shape[1:]) + + return outs diff --git a/comfy/ldm/depth_anything_3/model.py b/comfy/ldm/depth_anything_3/model.py new file mode 100644 index 000000000..f3c8a5ee3 --- /dev/null +++ b/comfy/ldm/depth_anything_3/model.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +from typing import Dict, Optional, Sequence + +import torch +import torch.nn as nn + +from comfy.image_encoders.dino2 import Dinov2Model + +from .camera import CameraDec, CameraEnc +from .dpt import DPT, DualDPT +from .ray_pose import get_extrinsic_from_camray +from .transform import affine_inverse, pose_encoding_to_extri_intri + + +_HEAD_REGISTRY = { + "dpt": DPT, + "dualdpt": DualDPT, +} + + +# Backbone presets (mirror the upstream DINOv2 ViT variants). +_BACKBONE_PRESETS = { + "vits": dict(hidden_size=384, num_hidden_layers=12, num_attention_heads=6, use_swiglu_ffn=False), + "vitb": dict(hidden_size=768, num_hidden_layers=12, num_attention_heads=12, use_swiglu_ffn=False), + "vitl": dict(hidden_size=1024, num_hidden_layers=24, num_attention_heads=16, use_swiglu_ffn=False), + "vitg": dict(hidden_size=1536, num_hidden_layers=40, num_attention_heads=24, use_swiglu_ffn=True), +} + + +def _build_backbone_config( + backbone_name: str, + *, + alt_start: int, + qknorm_start: int, + rope_start: int, + cat_token: bool, +) -> dict: + if backbone_name not in _BACKBONE_PRESETS: + raise ValueError(f"Unknown DINOv2 backbone variant: {backbone_name!r}") + cfg = dict(_BACKBONE_PRESETS[backbone_name]) + cfg.update(dict( + layer_norm_eps=1e-6, + patch_size=14, + image_size=518, + # No mask_token in DA3 weights; omit param to avoid load warnings. + use_mask_token=False, + alt_start=alt_start, + qknorm_start=qknorm_start, + rope_start=rope_start, + cat_token=cat_token, + rope_freq=100.0, + )) + return cfg + + +class DepthAnything3Net(nn.Module): + + PATCH_SIZE = 14 + + def __init__( + self, + # --- Backbone --- + backbone_name: str = "vitl", + out_layers: Sequence[int] = (4, 11, 17, 23), + alt_start: int = -1, + qknorm_start: int = -1, + rope_start: int = -1, + cat_token: bool = False, + # --- Head --- + head_type: str = "dpt", # dpt or dualdpt + head_dim_in: int = 1024, + head_output_dim: int = 1, # 1 = depth only, 2 = depth+conf + head_features: int = 256, + head_out_channels: Sequence[int] = (256, 512, 1024, 1024), + head_use_sky_head: bool = True, # ignored by DualDPT + head_pos_embed: Optional[bool] = None, # default: True for DualDPT, False for DPT + # --- Camera (multi-view) --- + has_cam_enc: bool = False, + has_cam_dec: bool = False, + cam_dim_out: Optional[int] = None, # CameraEnc dim_out (defaults to embed_dim) + cam_dec_dim_in: Optional[int] = None, # CameraDec dim_in (defaults to 2*embed_dim with cat_token) + # ComfyUI plumbing + device=None, dtype=None, operations=None, + **_ignored, + ): + super().__init__() + head_cls = _HEAD_REGISTRY[head_type.lower()] + self.head_type = head_type.lower() + self.has_sky = (self.head_type == "dpt") and head_use_sky_head + self.has_conf = head_output_dim > 1 + self.out_layers = list(out_layers) + + backbone_cfg = _build_backbone_config( + backbone_name, + alt_start=alt_start, + qknorm_start=qknorm_start, + rope_start=rope_start, + cat_token=cat_token, + ) + self.backbone = Dinov2Model(backbone_cfg, dtype, device, operations) + + head_kwargs = dict( + dim_in=head_dim_in, + patch_size=self.PATCH_SIZE, + output_dim=head_output_dim, + features=head_features, + out_channels=tuple(head_out_channels), + device=device, dtype=dtype, operations=operations, + ) + if self.head_type == "dpt": + head_kwargs.update( + use_sky_head=head_use_sky_head, + pos_embed=(False if head_pos_embed is None else head_pos_embed), + ) + else: # dualdpt + head_kwargs.update( + pos_embed=(True if head_pos_embed is None else head_pos_embed), + ) + self.head = head_cls(**head_kwargs) + + # Built only if checkpoint has weights; cam_enc output dim == embed_dim. + embed_dim = backbone_cfg["hidden_size"] + if has_cam_enc: + self.cam_enc = CameraEnc( + dim_out=cam_dim_out if cam_dim_out is not None else embed_dim, + num_heads=max(1, embed_dim // 64), + device=device, dtype=dtype, operations=operations, + ) + else: + self.cam_enc = None + if has_cam_dec: + default_dim = embed_dim * (2 if cat_token else 1) + self.cam_dec = CameraDec( + dim_in=cam_dec_dim_in if cam_dec_dim_in is not None else default_dim, + device=device, dtype=dtype, operations=operations, + ) + else: + self.cam_dec = None + + self.dtype = dtype + + def forward( + self, + image: torch.Tensor, + extrinsics: Optional[torch.Tensor] = None, + intrinsics: Optional[torch.Tensor] = None, + *, + use_ray_pose: bool = False, + ref_view_strategy: str = "saddle_balanced", + export_feat_layers: Optional[Sequence[int]] = None, + **_unused, + ) -> Dict[str, torch.Tensor]: + """Run depth and optionally pose prediction.""" + if image.ndim == 4: + image = image.unsqueeze(1) # (B, 1, 3, H, W) + assert image.ndim == 5 and image.shape[2] == 3, \ + f"image must be (B,3,H,W) or (B,S,3,H,W); got {tuple(image.shape)}" + + B, S, _, H, W = image.shape + assert H % self.PATCH_SIZE == 0 and W % self.PATCH_SIZE == 0, \ + f"image H,W must be multiples of {self.PATCH_SIZE}; got {(H, W)}" + + # Camera-token preparation (multi-view path). + cam_token = None + if extrinsics is not None and intrinsics is not None and self.cam_enc is not None: + cam_token = self.cam_enc(extrinsics, intrinsics, (H, W)) + + # Toggle aux ray output on/off depending on what the caller asked for. + if isinstance(self.head, DualDPT): + self.head.enable_aux = bool(use_ray_pose) + + feats, aux_feats = self.backbone.get_intermediate_layers_da3( + image, self.out_layers, cam_token=cam_token, + ref_view_strategy=ref_view_strategy, + export_feat_layers=export_feat_layers, + ) + head_out = self.head(feats, H=H, W=W, patch_start_idx=0) + + # Pose prediction. + out: Dict[str, torch.Tensor] = {} + if use_ray_pose and "ray" in head_out and "ray_conf" in head_out: + ray = head_out["ray"] + ray_conf = head_out["ray_conf"] + extr_c2w, focal, pp = get_extrinsic_from_camray( + ray, ray_conf, ray.shape[-3], ray.shape[-2], + ) + # Match the upstream output: w2c, drop the homogeneous row. + extr_w2c = affine_inverse(extr_c2w)[:, :, :3, :] + # Build pixel-space intrinsics from the normalised focal/pp output. + intr = torch.eye(3, device=ray.device, dtype=ray.dtype) + intr = intr[None, None].expand(extr_c2w.shape[0], extr_c2w.shape[1], 3, 3).clone() + intr[:, :, 0, 0] = focal[:, :, 0] / 2 * W + intr[:, :, 1, 1] = focal[:, :, 1] / 2 * H + intr[:, :, 0, 2] = pp[:, :, 0] * W * 0.5 + intr[:, :, 1, 2] = pp[:, :, 1] * H * 0.5 + out["extrinsics"] = extr_w2c + out["intrinsics"] = intr + elif self.cam_dec is not None and S > 1: + # Decode the cam-token of the final out_layer into a pose encoding. + cam_feat = feats[-1][1] # (B, S, dim_in_to_cam_dec) + pose_enc = self.cam_dec(cam_feat) + c2w_3x4, intr = pose_encoding_to_extri_intri(pose_enc, (H, W)) + # Match the upstream output convention: w2c (world->camera), 3x4. + c2w_4x4 = torch.cat([ + c2w_3x4, + torch.tensor([0, 0, 0, 1], device=c2w_3x4.device, dtype=c2w_3x4.dtype) + .view(1, 1, 1, 4).expand(B, S, 1, 4), + ], dim=-2) + out["extrinsics"] = affine_inverse(c2w_4x4)[:, :, :3, :] + out["intrinsics"] = intr + + # Flatten the views axis for per-pixel outputs (depth/conf/sky) so the + # per-image consumer keeps its (B*S, H, W) interface. + for k, v in head_out.items(): + if k in ("ray", "ray_conf"): + # Keep multi-view shape for downstream pose work. + out[k] = v + elif v.ndim >= 3 and v.shape[0] == B and v.shape[1] == S: + out[k] = v.reshape(B * S, *v.shape[2:]) + else: + out[k] = v + + if export_feat_layers: + out["aux_features"] = self._reshape_aux_features(aux_feats, H, W) + return out + + def _reshape_aux_features(self, aux_feats, H: int, W: int): + """Reshape (B, S, N, C) aux features into (B, S, h_p, w_p, C).""" + ph, pw = H // self.PATCH_SIZE, W // self.PATCH_SIZE + out = [] + for f in aux_feats: + B, S, N, C = f.shape + assert N == ph * pw, f"aux feature seq mismatch: {N} != {ph}*{pw}" + out.append(f.reshape(B, S, ph, pw, C)) + return out diff --git a/comfy/ldm/depth_anything_3/preprocess.py b/comfy/ldm/depth_anything_3/preprocess.py new file mode 100644 index 000000000..2238bd0d6 --- /dev/null +++ b/comfy/ldm/depth_anything_3/preprocess.py @@ -0,0 +1,128 @@ +"""Input/output preprocessing helpers for Depth Anything 3.""" + +from __future__ import annotations + +from typing import Tuple + +import torch + +import comfy.utils + +PATCH_SIZE = 14 + +# ImageNet normalization constants used during DA3 training. +_IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]) +_IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]) + + +def _round_to_patch(x: int, patch: int = PATCH_SIZE) -> int: + down = (x // patch) * patch + up = down + patch + return up if abs(up - x) <= abs(x - down) else down + + +def compute_target_size(orig_h: int, orig_w: int, process_res: int, method: str = "upper_bound_resize") -> Tuple[int, int]: + """Compute (target_h, target_w) for a single image. + upper_bound_resize: scale longest side to process_res, then round each dim to nearest multiple of 14 (default upstream method). + lower_bound_resize: scale shortest side to process_res, then round.""" + + if method == "upper_bound_resize": + longest = max(orig_h, orig_w) + scale = process_res / float(longest) + elif method == "lower_bound_resize": + shortest = min(orig_h, orig_w) + scale = process_res / float(shortest) + else: + raise ValueError(f"Unsupported process_res_method: {method}") + + new_w = max(1, _round_to_patch(int(round(orig_w * scale)))) + new_h = max(1, _round_to_patch(int(round(orig_h * scale)))) + return new_h, new_w + + +def preprocess_image(image: torch.Tensor, process_res: int = 504, method: str = "upper_bound_resize") -> torch.Tensor: + assert image.ndim == 4 and image.shape[-1] == 3, f"expected (B,H,W,3) IMAGE; got {tuple(image.shape)}" + B, H, W, _ = image.shape + target_h, target_w = compute_target_size(H, W, process_res, method) + + # (B, H, W, 3) -> (B, 3, H, W) + x = image.movedim(-1, 1).contiguous() + if (target_h, target_w) != (H, W): + # Upstream uses cv2 INTER_CUBIC (upscale) / INTER_AREA (downscale). + # Lanczos in ``common_upscale`` is anti-aliased and produces the + # closest pixel-wise match in a sweep across {bilinear, bicubic, + # area, lanczos, bislerp}. Used in both directions for simplicity. + x = comfy.utils.common_upscale(x.float(), target_w, target_h, "lanczos", "disabled",) + x = x.clamp(0.0, 1.0) + + mean = _IMAGENET_MEAN.to(device=x.device, dtype=x.dtype).view(1, 3, 1, 1) + std = _IMAGENET_STD.to(device=x.device, dtype=x.dtype).view(1, 3, 1, 1) + x = (x - mean) / std + return x + + +# ----------------------------------------------------------------------------- +# Output post-processing (sky-aware clipping for Mono/Metric variants) +# ----------------------------------------------------------------------------- + + +def compute_non_sky_mask(sky_prediction: torch.Tensor, threshold: float = 0.3) -> torch.Tensor: + """Boolean mask: True for non-sky pixels (sky probability < threshold).""" + return sky_prediction < threshold + + +def apply_sky_aware_clip(depth: torch.Tensor, sky: torch.Tensor, threshold: float = 0.3, quantile: float = 0.99) -> torch.Tensor: + """Clips sky regions to the 99th percentile of non-sky depth. Returns a new depth tensor.""" + non_sky = compute_non_sky_mask(sky, threshold=threshold) + if non_sky.sum() <= 10 or (~non_sky).sum() <= 10: + return depth.clone() + + non_sky_depth = depth[non_sky] + if non_sky_depth.numel() > 100_000: + idx = torch.randint(0, non_sky_depth.numel(), (100_000,), device=non_sky_depth.device) + sampled = non_sky_depth[idx] + else: + sampled = non_sky_depth + + max_depth = torch.quantile(sampled, quantile) + out = depth.clone() + out[~non_sky] = max_depth + return out + + +def normalize_depth_v2_style(depth: torch.Tensor, sky: torch.Tensor | None = None, low_quantile: float = 0.01, high_quantile: float = 0.99) -> torch.Tensor: + """V2-style normalization computes percentile bounds over non-sky pixels (when available), then maps depth into [0, 1] with near = white (1.0).""" + if sky is not None: + mask = compute_non_sky_mask(sky) + if mask.any(): + valid = depth[mask] + else: + valid = depth.flatten() + else: + valid = depth.flatten() + + if valid.numel() > 100_000: + idx = torch.randint(0, valid.numel(), (100_000,), device=valid.device) + sample = valid[idx] + else: + sample = valid + + lo = torch.quantile(sample, low_quantile) + hi = torch.quantile(sample, high_quantile) + rng = (hi - lo).clamp(min=1e-6) + norm = ((depth - lo) / rng).clamp(0.0, 1.0) + # Nearer pixels are brighter (1.0) + norm = 1.0 - norm + if sky is not None: + # Sky pixels become black (far / unknown) + sky_mask = ~compute_non_sky_mask(sky) + norm = torch.where(sky_mask, torch.zeros_like(norm), norm) + return norm + + +def normalize_depth_min_max(depth: torch.Tensor) -> torch.Tensor: + """Simple per-frame min/max normalization with near=1.0 convention.""" + lo = depth.amin(dim=(-2, -1), keepdim=True) + hi = depth.amax(dim=(-2, -1), keepdim=True) + rng = (hi - lo).clamp(min=1e-6) + return 1.0 - ((depth - lo) / rng).clamp(0.0, 1.0) diff --git a/comfy/ldm/depth_anything_3/ray_pose.py b/comfy/ldm/depth_anything_3/ray_pose.py new file mode 100644 index 000000000..90890f1da --- /dev/null +++ b/comfy/ldm/depth_anything_3/ray_pose.py @@ -0,0 +1,272 @@ +"""Ray-to-pose conversion for the multi-view path of Depth Anything 3.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch + + +# qr/svd use fp32: CUDA often has no fp16/bf16 kernels for these ops. + + +def _ql_decomposition(A: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Decompose A = Q @ L with Q orthogonal and L lower-triangular. + Implemented in terms of QR by reversing the columns/rows; the standard + trick from the upstream reference. Inputs A are (3, 3).""" + P = torch.tensor([[0, 0, 1], [0, 1, 0], [1, 0, 0]], device=A.device, dtype=A.dtype) + A_tilde = A @ P + # CUDA QR is not implemented for fp16/bf16; upcast just for this call. + Q_tilde, R_tilde = torch.linalg.qr(A_tilde.float()) + Q_tilde = Q_tilde.to(A.dtype) + R_tilde = R_tilde.to(A.dtype) + Q = Q_tilde @ P + L = P @ R_tilde @ P + d = torch.diag(L) + sign = torch.sign(d) + Q = Q * sign[None, :] # scale columns of Q + L = L * sign[:, None] # scale rows of L + return Q, L + + +def _homogenize_points(points: torch.Tensor) -> torch.Tensor: + return torch.cat([points, torch.ones_like(points[..., :1])], dim=-1) + + +# ----------------------------------------------------------------------------- +# Weighted-LSQ + RANSAC homography (batched) +# ----------------------------------------------------------------------------- + + +def _find_homography_weighted_lsq(src_pts: torch.Tensor, dst_pts: torch.Tensor, confident_weight: torch.Tensor,) -> torch.Tensor: + """Solve a single H with weighted least-squares (DLT).""" + N = src_pts.shape[0] + if N < 4: + raise ValueError("At least 4 points are required to compute a homography.") + w = confident_weight.sqrt().unsqueeze(1) # (N, 1) + x = src_pts[:, 0:1] + y = src_pts[:, 1:2] + u = dst_pts[:, 0:1] + v = dst_pts[:, 1:2] + zeros = torch.zeros_like(x) + A1 = torch.cat([-x * w, -y * w, -w, zeros, zeros, zeros, x * u * w, y * u * w, u * w], dim=1) + A2 = torch.cat([zeros, zeros, zeros, -x * w, -y * w, -w, x * v * w, y * v * w, v * w], dim=1) + A = torch.cat([A1, A2], dim=0) # (2N, 9) + # CUDA SVD is not implemented for fp16/bf16; upcast just for this call. + _, _, Vh = torch.linalg.svd(A.float()) + Vh = Vh.to(A.dtype) + H = Vh[-1].reshape(3, 3) + return H / H[-1, -1] + + +def _find_homography_weighted_lsq_batched(src_pts_batch: torch.Tensor, dst_pts_batch: torch.Tensor, confident_weight_batch: torch.Tensor) -> torch.Tensor: + """Batched DLT solver. Inputs (B, K, 2) / (B, K); output (B, 3, 3).""" + B, K, _ = src_pts_batch.shape + w = confident_weight_batch.sqrt().unsqueeze(2) + x = src_pts_batch[:, :, 0:1] + y = src_pts_batch[:, :, 1:2] + u = dst_pts_batch[:, :, 0:1] + v = dst_pts_batch[:, :, 1:2] + zeros = torch.zeros_like(x) + A1 = torch.cat([-x * w, -y * w, -w, zeros, zeros, zeros, x * u * w, y * u * w, u * w], dim=2) + A2 = torch.cat([zeros, zeros, zeros, -x * w, -y * w, -w, x * v * w, y * v * w, v * w], dim=2) + A = torch.cat([A1, A2], dim=1) # (B, 2K, 9) + # CUDA SVD is not implemented for fp16/bf16; upcast just for this call. + _, _, Vh = torch.linalg.svd(A.float()) + Vh = Vh.to(A.dtype) + H = Vh[:, -1].reshape(B, 3, 3) + return H / H[:, 2:3, 2:3] + + +def _ransac_find_homography_weighted_batched( + src_pts: torch.Tensor, # (B, N, 2) + dst_pts: torch.Tensor, # (B, N, 2) + confident_weight: torch.Tensor, # (B, N) + n_sample: int, + n_iter: int = 100, + reproj_threshold: float = 3.0, + num_sample_for_ransac: int = 8, + max_inlier_num: int = 10000, + rand_sample_iters_idx: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Batched weighted-RANSAC homography estimator. Returns (B, 3, 3) homography matrices.""" + B, N, _ = src_pts.shape + assert N >= 4 + device = src_pts.device + + sorted_idx = torch.argsort(confident_weight, descending=True, dim=1) + candidate_idx = sorted_idx[:, :n_sample] # (B, n_sample) + + if rand_sample_iters_idx is None: + rand_sample_iters_idx = torch.stack( + [torch.randperm(n_sample, device=device)[:num_sample_for_ransac] + for _ in range(n_iter)], + dim=0, + ) + + rand_idx = candidate_idx[:, rand_sample_iters_idx] # (B, n_iter, k) + b_idx = ( + torch.arange(B, device=device) + .view(B, 1, 1) + .expand(B, n_iter, num_sample_for_ransac) + ) + src_b = src_pts[b_idx, rand_idx] + dst_b = dst_pts[b_idx, rand_idx] + w_b = confident_weight[b_idx, rand_idx] + + cB, cN = src_b.shape[:2] + H_batch = _find_homography_weighted_lsq_batched( + src_b.flatten(0, 1), dst_b.flatten(0, 1), w_b.flatten(0, 1), + ).unflatten(0, (cB, cN)) # (B, n_iter, 3, 3) + + src_homo = torch.cat([src_pts, torch.ones(B, N, 1, device=device, dtype=src_pts.dtype)], dim=2) + proj = torch.bmm( + src_homo.unsqueeze(1).expand(B, n_iter, N, 3).reshape(-1, N, 3), + H_batch.reshape(-1, 3, 3).transpose(1, 2), + ) # (B*n_iter, N, 3) + proj_xy = (proj[:, :, :2] / proj[:, :, 2:3]).reshape(B, n_iter, N, 2) + err = ((proj_xy - dst_pts.unsqueeze(1)) ** 2).sum(-1).sqrt() # (B, n_iter, N) + inlier_mask = err < reproj_threshold + score = (inlier_mask * confident_weight.unsqueeze(1)).sum(dim=2) + best_idx = torch.argmax(score, dim=1) + best_inlier_mask = inlier_mask[torch.arange(B, device=device), best_idx] + + # Refit with the inlier set (per-batch, since the inlier counts vary). + H_inlier_list = [] + for b in range(B): + mask = best_inlier_mask[b] + in_src = src_pts[b][mask] + in_dst = dst_pts[b][mask] + in_w = confident_weight[b][mask] + if in_src.shape[0] < 4: + # Fall back to identity when RANSAC fails to find enough inliers. + H_inlier_list.append(torch.eye(3, device=device, dtype=src_pts.dtype)) + continue + sorted_w = torch.argsort(in_w, descending=True) + if len(sorted_w) > max_inlier_num: + keep = max(int(len(sorted_w) * 0.95), max_inlier_num) + sorted_w = sorted_w[:keep][torch.randperm(keep, device=device)[:max_inlier_num]] + H_inlier_list.append( + _find_homography_weighted_lsq(in_src[sorted_w], in_dst[sorted_w], in_w[sorted_w]) + ) + return torch.stack(H_inlier_list, dim=0) + + +# ----------------------------------------------------------------------------- +# Camera-ray utilities +# ----------------------------------------------------------------------------- + + +def _unproject_identity(num_y: int, num_x: int, B: int, S: int, device, dtype) -> torch.Tensor: + """Camera-space unit rays for an identity intrinsic on a 2x2 image plane.""" + dx = 1.0 / num_x + dy = 1.0 / num_y + # Centered camera-space coords directly (skip the K^-1 step since it's + # just a translation by -1 on x and y when K is identity-with-center=1). + y = torch.linspace(-(1 - dy), (1 - dy), num_y, device=device, dtype=dtype) + x = torch.linspace(-(1 - dx), (1 - dx), num_x, device=device, dtype=dtype) + yy, xx = torch.meshgrid(y, x, indexing="ij") + grid = torch.stack((xx, yy), dim=-1) # (h, w, 2) + grid = grid.unsqueeze(0).unsqueeze(0).expand(B, S, num_y, num_x, 2) + return torch.cat([grid, torch.ones_like(grid[..., :1])], dim=-1) + + +def _camray_to_caminfo( + camray: torch.Tensor, # (B, S, h, w, 6) + confidence: Optional[torch.Tensor] = None, # (B, S, h, w) + reproj_threshold: float = 0.2, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Convert per-pixel camera rays to per-view (R, T, focal, principal).""" + if confidence is None: + confidence = torch.ones_like(camray[..., 0]) + B, S, h, w, _ = camray.shape + device = camray.device + dtype = camray.dtype + + rays_target = camray[..., :3] # (B, S, h, w, 3) + rays_origin = _unproject_identity(h, w, B, S, device, dtype) + + # Flatten (B*S, h*w, *) for the RANSAC routine. + rays_target = rays_target.flatten(0, 1).flatten(1, 2) + rays_origin = rays_origin.flatten(0, 1).flatten(1, 2) + weights = confidence.flatten(0, 1).flatten(1, 2).clone() + + # Project to 2D in homogeneous form (the upstream calls this "perspective division"). + z_thresh = 1e-4 + mask = (rays_target[:, :, 2].abs() > z_thresh) & (rays_origin[:, :, 2].abs() > z_thresh) + weights = torch.where(mask, weights, torch.zeros_like(weights)) + src = rays_origin.clone() + dst = rays_target.clone() + src[..., 0] = torch.where(mask, src[..., 0] / src[..., 2], src[..., 0]) + src[..., 1] = torch.where(mask, src[..., 1] / src[..., 2], src[..., 1]) + dst[..., 0] = torch.where(mask, dst[..., 0] / dst[..., 2], dst[..., 0]) + dst[..., 1] = torch.where(mask, dst[..., 1] / dst[..., 2], dst[..., 1]) + src = src[..., :2] + dst = dst[..., :2] + + N = src.shape[1] + n_iter = 100 + sample_ratio = 0.3 + num_sample_for_ransac = 8 + n_sample = max(num_sample_for_ransac, int(N * sample_ratio)) + rand_idx = torch.stack( + [torch.randperm(n_sample, device=device)[:num_sample_for_ransac] for _ in range(n_iter)], + dim=0, + ) + + # Chunk along the view axis to keep peak memory predictable. + chunk = 2 + A_list = [] + for i in range(0, src.shape[0], chunk): + A = _ransac_find_homography_weighted_batched( + src[i:i + chunk], dst[i:i + chunk], weights[i:i + chunk], + n_sample=n_sample, n_iter=n_iter, + num_sample_for_ransac=num_sample_for_ransac, + reproj_threshold=reproj_threshold, + rand_sample_iters_idx=rand_idx, + max_inlier_num=8000, + ) + # Flip sign on dets that come out < 0 (so that the QL produces a + # right-handed rotation). ``det`` lacks fp16/bf16 CUDA kernels, so + # do the comparison in fp32. + flip = torch.linalg.det(A.float()) < 0 + A = torch.where(flip[:, None, None], -A, A) + A_list.append(A) + A = torch.cat(A_list, dim=0) # (B*S, 3, 3) + + R_list, f_list, pp_list = [], [], [] + for i in range(A.shape[0]): + R, L = _ql_decomposition(A[i]) + L = L / L[2][2] + f_list.append(torch.stack((L[0][0], L[1][1]))) + pp_list.append(torch.stack((L[2][0], L[2][1]))) + R_list.append(R) + R = torch.stack(R_list).reshape(B, S, 3, 3) + focal = torch.stack(f_list).reshape(B, S, 2) + pp = torch.stack(pp_list).reshape(B, S, 2) + + # Translation: confidence-weighted average of camray direction(s). + cf = confidence.flatten(0, 1).flatten(1, 2) + T = (camray.flatten(0, 1).flatten(1, 2)[..., 3:] * cf.unsqueeze(-1)).sum(dim=1) + T = T / cf.sum(dim=-1, keepdim=True) + T = T.reshape(B, S, 3) + + # Match upstream output convention: focal -> 1/focal, pp + 1. + return R, T, 1.0 / focal, pp + 1.0 + + +def get_extrinsic_from_camray( + camray: torch.Tensor, # (B, S, h, w, 6) + conf: torch.Tensor, # (B, S, h, w, 1) or (B, S, h, w) + patch_size_y: int, + patch_size_x: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Wrap a 4x4 extrinsic + per-view focal + principal-point output.""" + if conf.ndim == 5 and conf.shape[-1] == 1: + conf = conf.squeeze(-1) + R, T, focal, pp = _camray_to_caminfo(camray, confidence=conf) + extr = torch.cat([R, T.unsqueeze(-1)], dim=-1) # (B, S, 3, 4) + homo_row = torch.tensor([0, 0, 0, 1], dtype=R.dtype, device=R.device) + homo_row = homo_row.view(1, 1, 1, 4).expand(R.shape[0], R.shape[1], 1, 4) + extr = torch.cat([extr, homo_row], dim=-2) # (B, S, 4, 4) + return extr, focal, pp diff --git a/comfy/ldm/depth_anything_3/reference_view_selector.py b/comfy/ldm/depth_anything_3/reference_view_selector.py new file mode 100644 index 000000000..90f00be92 --- /dev/null +++ b/comfy/ldm/depth_anything_3/reference_view_selector.py @@ -0,0 +1,87 @@ +"""Reference-view selection for the multi-view path of Depth Anything 3.""" + +from __future__ import annotations + +from typing import Literal + +import torch + + +RefViewStrategy = Literal["first", "middle", "saddle_balanced", "saddle_sim_range"] + + +# Per the upstream constants module: ``THRESH_FOR_REF_SELECTION = 3``. +# Reference selection only runs when there are at least this many views. +THRESH_FOR_REF_SELECTION: int = 3 + + +def select_reference_view(x: torch.Tensor, strategy: RefViewStrategy = "saddle_balanced") -> torch.Tensor: + """Pick a reference view index per batch element.""" + B, S, _, _ = x.shape + if S <= 1: + return torch.zeros(B, dtype=torch.long, device=x.device) + if strategy == "first": + return torch.zeros(B, dtype=torch.long, device=x.device) + if strategy == "middle": + return torch.full((B,), S // 2, dtype=torch.long, device=x.device) + + # Feature-based strategies: normalised cls/cam token per view. + img_class_feat = x[:, :, 0] / x[:, :, 0].norm(dim=-1, keepdim=True) # (B,S,C) + + if strategy == "saddle_balanced": + sim = torch.matmul(img_class_feat, img_class_feat.transpose(1, 2)) # (B,S,S) + sim_no_diag = sim - torch.eye(S, device=sim.device).unsqueeze(0) + sim_score = sim_no_diag.sum(dim=-1) / (S - 1) # (B,S) + feat_norm = x[:, :, 0].norm(dim=-1) # (B,S) + feat_var = img_class_feat.var(dim=-1) # (B,S) + + def _normalize(metric): + mn = metric.min(dim=1, keepdim=True).values + mx = metric.max(dim=1, keepdim=True).values + return (metric - mn) / (mx - mn + 1e-8) + + sim_n, norm_n, var_n = _normalize(sim_score), _normalize(feat_norm), _normalize(feat_var) + balance = (sim_n - 0.5).abs() + (norm_n - 0.5).abs() + (var_n - 0.5).abs() + return balance.argmin(dim=1) + + if strategy == "saddle_sim_range": + sim = torch.matmul(img_class_feat, img_class_feat.transpose(1, 2)) + sim_no_diag = sim - torch.eye(S, device=sim.device).unsqueeze(0) + sim_max = sim_no_diag.max(dim=-1).values + sim_min = sim_no_diag.min(dim=-1).values + return (sim_max - sim_min).argmax(dim=1) + + raise ValueError( + f"Unknown reference view selection strategy: {strategy!r}. " + f"Must be one of: 'first', 'middle', 'saddle_balanced', 'saddle_sim_range'" + ) + + +def reorder_by_reference(x: torch.Tensor, b_idx: torch.Tensor) -> torch.Tensor: + """Reorder x so the reference view is at position 0 in axis S.""" + B, S = x.shape[0], x.shape[1] + if S <= 1: + return x + positions = torch.arange(S, device=x.device).unsqueeze(0).expand(B, -1) + b_idx_exp = b_idx.unsqueeze(1) + reorder = torch.where( + (positions > 0) & (positions <= b_idx_exp), + positions - 1, + positions, + ) + reorder[:, 0] = b_idx + batch = torch.arange(B, device=x.device).unsqueeze(1) + return x[batch, reorder] + + +def restore_original_order(x: torch.Tensor, b_idx: torch.Tensor) -> torch.Tensor: + """Inverse of reorder_by_reference.""" + B, S = x.shape[0], x.shape[1] + if S <= 1: + return x + target_positions = torch.arange(S, device=x.device).unsqueeze(0).expand(B, -1) + b_idx_exp = b_idx.unsqueeze(1) + restore = torch.where(target_positions < b_idx_exp, target_positions + 1, target_positions) + restore = torch.scatter(restore, dim=1, index=b_idx_exp, src=torch.zeros_like(b_idx_exp)) + batch = torch.arange(B, device=x.device).unsqueeze(1) + return x[batch, restore] diff --git a/comfy/ldm/depth_anything_3/transform.py b/comfy/ldm/depth_anything_3/transform.py new file mode 100644 index 000000000..b735d7bec --- /dev/null +++ b/comfy/ldm/depth_anything_3/transform.py @@ -0,0 +1,160 @@ +"""Geometry / camera transform helpers for Depth Anything 3.""" + +from __future__ import annotations + +from typing import Tuple + +import torch +import torch.nn.functional as F + + +# ----------------------------------------------------------------------------- +# Affine 4x4 helpers +# ----------------------------------------------------------------------------- + + +def as_homogeneous(ext: torch.Tensor) -> torch.Tensor: + """Promote (...,3,4) extrinsics to (...,4,4) homogeneous form. No-op when the input is already ``(...,4,4)``.""" + if ext.shape[-2:] == (4, 4): + return ext + if ext.shape[-2:] == (3, 4): + ones = torch.zeros_like(ext[..., :1, :4]) + ones[..., 0, 3] = 1.0 + return torch.cat([ext, ones], dim=-2) + raise ValueError(f"Invalid affine shape: {ext.shape}") + + +def affine_inverse(A: torch.Tensor) -> torch.Tensor: + """Inverse of an affine matrix ``[R|T; 0 0 0 1]``.""" + R = A[..., :3, :3] + T = A[..., :3, 3:] + P = A[..., 3:, :] + return torch.cat([torch.cat([R.mT, -R.mT @ T], dim=-1), P], dim=-2) + + +# ----------------------------------------------------------------------------- +# Quaternion <-> rotation matrix (xyzw / scalar-last) +# ----------------------------------------------------------------------------- + + +def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: + """sqrt(max(0, x)) with a zero subgradient where x == 0.""" + ret = torch.zeros_like(x) + positive_mask = x > 0 + if torch.is_grad_enabled(): + ret[positive_mask] = torch.sqrt(x[positive_mask]) + else: + ret = torch.where(positive_mask, torch.sqrt(x), ret) + return ret + + +def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor: + """Force the real part of a unit quaternion (xyzw) to be non-negative.""" + return torch.where(quaternions[..., 3:4] < 0, -quaternions, quaternions) + + +def quat_to_mat(quaternions: torch.Tensor) -> torch.Tensor: + """Convert quaternions (xyzw) to (...,3,3) rotation matrices.""" + i, j, k, r = torch.unbind(quaternions, -1) + two_s = 2.0 / (quaternions * quaternions).sum(-1) + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return o.reshape(quaternions.shape[:-1] + (3, 3)) + + +def mat_to_quat(matrix: torch.Tensor) -> torch.Tensor: + """Convert (...,3,3) rotation matrices to quaternions (xyzw).""" + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + + batch_dim = matrix.shape[:-2] + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( + matrix.reshape(batch_dim + (9,)), dim=-1 + ) + + q_abs = _sqrt_positive_part( + torch.stack( + [ + 1.0 + m00 + m11 + m22, + 1.0 + m00 - m11 - m22, + 1.0 - m00 + m11 - m22, + 1.0 - m00 - m11 + m22, + ], + dim=-1, + ) + ) + + quat_by_rijk = torch.stack( + [ + torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), + torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), + torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), + torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), + ], + dim=-2, + ) + + flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) + quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) + + out = quat_candidates[F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, :].reshape( + batch_dim + (4,) + ) + # Reorder rijk -> xyzw (i.e. ijkr). + out = out[..., [1, 2, 3, 0]] + return standardize_quaternion(out) + + +# ----------------------------------------------------------------------------- +# Pose-encoding <-> extrinsics + intrinsics +# ----------------------------------------------------------------------------- + + +def extri_intri_to_pose_encoding(extrinsics: torch.Tensor, intrinsics: torch.Tensor, image_size_hw: Tuple[int, int]) -> torch.Tensor: + """Pack (extr, intr, image_size) into the 9-D pose-encoding vector. + extrinsics: camera-to-world (c2w) (B,S,4,4) matrices, + intrinsics: pixel-space (B,S,3,3) matrices, + image_size_hw: is a (H, W) pair. + """ + R = extrinsics[..., :3, :3] + T = extrinsics[..., :3, 3] + quat = mat_to_quat(R) + H, W = image_size_hw + fov_h = 2 * torch.atan((H / 2) / intrinsics[..., 1, 1]) + fov_w = 2 * torch.atan((W / 2) / intrinsics[..., 0, 0]) + return torch.cat([T, quat, fov_h[..., None], fov_w[..., None]], dim=-1).float() + + +def pose_encoding_to_extri_intri(pose_encoding: torch.Tensor, image_size_hw: Tuple[int, int]) -> Tuple[torch.Tensor, torch.Tensor]: + """Inverse of extri_intri_to_pose_encoding.""" + T = pose_encoding[..., :3] + quat = pose_encoding[..., 3:7] + fov_h = pose_encoding[..., 7] + fov_w = pose_encoding[..., 8] + # Normalize to unit quaternion. CameraDec outputs raw values; a near-zero + # quaternion causes two_s = 2/norm² → inf in quat_to_mat → NaN extrinsics. + quat = quat / quat.norm(dim=-1, keepdim=True).clamp(min=1e-6) + R = quat_to_mat(quat) + extrinsics = torch.cat([R, T[..., None]], dim=-1) + H, W = image_size_hw + fy = (H / 2.0) / torch.clamp(torch.tan(fov_h / 2.0), 1e-6) + fx = (W / 2.0) / torch.clamp(torch.tan(fov_w / 2.0), 1e-6) + intrinsics = torch.zeros(pose_encoding.shape[:2] + (3, 3), device=pose_encoding.device, dtype=pose_encoding.dtype) + intrinsics[..., 0, 0] = fx + intrinsics[..., 1, 1] = fy + intrinsics[..., 0, 2] = W / 2 + intrinsics[..., 1, 2] = H / 2 + intrinsics[..., 2, 2] = 1.0 + return extrinsics, intrinsics diff --git a/comfy/ldm/flux/math.py b/comfy/ldm/flux/math.py index 6d0aed827..891dea7dd 100644 --- a/comfy/ldm/flux/math.py +++ b/comfy/ldm/flux/math.py @@ -4,7 +4,7 @@ from torch import Tensor from comfy.ldm.modules.attention import optimized_attention import comfy.model_management -import logging +import comfy.quant_ops def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, mask=None, transformer_options={}) -> Tensor: @@ -44,21 +44,15 @@ def _apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor): return apply_rope1(xq, freqs_cis), apply_rope1(xk, freqs_cis) -try: - import comfy.quant_ops - q_apply_rope = comfy.quant_ops.ck.apply_rope - q_apply_rope1 = comfy.quant_ops.ck.apply_rope1 - def apply_rope(xq, xk, freqs_cis): - if comfy.model_management.in_training: - return _apply_rope(xq, xk, freqs_cis) - else: - return apply_rope1(xq, freqs_cis), apply_rope1(xk, freqs_cis) - def apply_rope1(x, freqs_cis): - if comfy.model_management.in_training: - return _apply_rope1(x, freqs_cis) - else: - return q_apply_rope1(x, freqs_cis) -except: - logging.warning("No comfy kitchen, using old apply_rope functions.") - apply_rope = _apply_rope - apply_rope1 = _apply_rope1 +def apply_rope(xq, xk, freqs_cis): + if comfy.model_management.in_training: + return _apply_rope(xq, xk, freqs_cis) + else: + return comfy.quant_ops.ck.apply_rope(xq, xk, freqs_cis) + + +def apply_rope1(x, freqs_cis): + if comfy.model_management.in_training: + return _apply_rope1(x, freqs_cis) + else: + return comfy.quant_ops.ck.apply_rope1(x, freqs_cis) diff --git a/comfy/ldm/hidream_o1/attention.py b/comfy/ldm/hidream_o1/attention.py index 1b68f1771..afb2be9b8 100644 --- a/comfy/ldm/hidream_o1/attention.py +++ b/comfy/ldm/hidream_o1/attention.py @@ -15,24 +15,24 @@ def make_two_pass_attention(ar_len: int, transformer_options=None): The AR pass goes through SDPA directand bypasses wrappers, it is only ~1% of T at typical edit sizes. """ - def two_pass_attention(q, k, v, heads, **kwargs): + def two_pass_attention(q, k, v, heads, enable_gqa=False, **kwargs): B, H, T, D = q.shape if T < k.shape[2]: # KV-cache hot path: Q is shorter than K/V (cached AR prefix is in K/V only), all fresh Q positions are in the gen region, single full-attention call - out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options) + out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, enable_gqa=enable_gqa) elif ar_len >= T: - out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True) + out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True, enable_gqa=enable_gqa) elif ar_len <= 0: - out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options) + out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, enable_gqa=enable_gqa) else: out_ar = comfy.ops.scaled_dot_product_attention( q[:, :, :ar_len], k[:, :, :ar_len], v[:, :, :ar_len], - attn_mask=None, dropout_p=0.0, is_causal=True, + attn_mask=None, dropout_p=0.0, is_causal=True, enable_gqa=enable_gqa, ) out_gen = optimized_attention( q[:, :, ar_len:], k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, - transformer_options=transformer_options, + transformer_options=transformer_options, enable_gqa=enable_gqa, ) out = torch.cat([out_ar, out_gen], dim=2) diff --git a/comfy/ldm/ideogram4/model.py b/comfy/ldm/ideogram4/model.py new file mode 100644 index 000000000..4ea5b8aaf --- /dev/null +++ b/comfy/ldm/ideogram4/model.py @@ -0,0 +1,297 @@ +""" +The Ideogram 4 transformer is a NextDiT/Lumina2-family single-stream model +consumes Qwen3-VL hidden-state features (concatenated from 13 layers -> 53248 dims) +packs ``[text tokens, image tokens]`` into one sequence with block-diagonal segment attention and 3D interleaved MRoPE. +""" + +from __future__ import annotations + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.patcher_extension +from comfy.ldm.lumina.model import FeedForward +from comfy.ldm.modules.attention import optimized_attention_masked +from comfy.text_encoders.llama import apply_rope, precompute_freqs_cis + +# Per-token role indicators +SEQUENCE_PADDING_INDICATOR = -1 +OUTPUT_IMAGE_INDICATOR = 2 +LLM_TOKEN_INDICATOR = 3 +# Image grid coordinates are offset so they never collide with text positions +IMAGE_POSITION_OFFSET = 65536 + + +class Ideogram4Attention(nn.Module): + def __init__(self, hidden_size, num_heads, eps=1e-5, dtype=None, device=None, operations=None): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.hidden_size = hidden_size + + self.qkv = operations.Linear(hidden_size, hidden_size * 3, bias=False, dtype=dtype, device=device) + self.norm_q = operations.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, dtype=dtype, device=device) + self.norm_k = operations.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, dtype=dtype, device=device) + self.o = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device) + + def forward(self, x, attn_mask, freqs_cis, transformer_options={}): + batch_size, seq_len, _ = x.shape + qkv = self.qkv(x).view(batch_size, seq_len, 3, self.num_heads, self.head_dim) + q, k, v = qkv.unbind(dim=2) + + q = self.norm_q(q) + k = self.norm_k(k) + + # (B, heads, L, head_dim) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + q, k = apply_rope(q, k, freqs_cis) + + out = optimized_attention_masked(q, k, v, self.num_heads, attn_mask, skip_reshape=True, transformer_options=transformer_options) + return self.o(out) + + +class Ideogram4TransformerBlock(nn.Module): + def __init__(self, hidden_size, intermediate_size, num_heads, norm_eps, adaln_dim, dtype=None, device=None, operations=None): + super().__init__() + self.attention = Ideogram4Attention(hidden_size, num_heads, eps=1e-5, dtype=dtype, device=device, operations=operations) + self.feed_forward = FeedForward( + dim=hidden_size, hidden_dim=intermediate_size, multiple_of=1, ffn_dim_multiplier=None, + operation_settings={"operations": operations, "dtype": dtype, "device": device}, + ) + + self.attention_norm1 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device) + self.ffn_norm1 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device) + self.attention_norm2 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device) + self.ffn_norm2 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device) + + self.adaln_modulation = operations.Linear(adaln_dim, 4 * hidden_size, bias=True, dtype=dtype, device=device) + + def forward(self, x, attn_mask, freqs_cis, adaln_input, transformer_options={}): + mod = self.adaln_modulation(adaln_input) + scale_msa, gate_msa, scale_mlp, gate_mlp = mod.chunk(4, dim=-1) + gate_msa = torch.tanh(gate_msa) + gate_mlp = torch.tanh(gate_mlp) + scale_msa = 1.0 + scale_msa + scale_mlp = 1.0 + scale_mlp + + attn_out = self.attention(self.attention_norm1(x) * scale_msa, attn_mask, freqs_cis, transformer_options=transformer_options) + x = x + gate_msa * self.attention_norm2(attn_out) + x = x + gate_mlp * self.ffn_norm2(self.feed_forward(self.ffn_norm1(x) * scale_mlp)) + return x + + +def _sinusoidal_embedding(t, dim, scale=1e4): + t = t.to(torch.float32) + half = dim // 2 + freq = math.log(scale) / (half - 1) + freq = torch.exp(torch.arange(half, dtype=torch.float32, device=t.device) * -freq) + emb = t.unsqueeze(-1) * freq + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1) + if dim % 2 == 1: + emb = F.pad(emb, (0, 1)) + return emb + + +class Ideogram4EmbedScalar(nn.Module): + def __init__(self, dim, input_range=(0.0, 1.0), dtype=None, device=None, operations=None): + super().__init__() + self.dim = dim + self.range_min, self.range_max = input_range + self.mlp_in = operations.Linear(dim, dim, bias=True, dtype=dtype, device=device) + self.mlp_out = operations.Linear(dim, dim, bias=True, dtype=dtype, device=device) + + def forward(self, x, dtype): + x = x.to(torch.float32) + scaled = 1e4 * (x - self.range_min) / (self.range_max - self.range_min) + emb = _sinusoidal_embedding(scaled, self.dim) + emb = emb.to(dtype) + emb = F.silu(self.mlp_in(emb)) + return self.mlp_out(emb) + + +class Ideogram4FinalLayer(nn.Module): + def __init__(self, hidden_size, out_channels, adaln_dim, dtype=None, device=None, operations=None): + super().__init__() + self.norm_final = operations.LayerNorm(hidden_size, eps=1e-6, elementwise_affine=False, dtype=dtype, device=device) + self.linear = operations.Linear(hidden_size, out_channels, bias=True, dtype=dtype, device=device) + self.adaln_modulation = operations.Linear(adaln_dim, hidden_size, bias=True, dtype=dtype, device=device) + + def forward(self, x, c): + scale = 1.0 + self.adaln_modulation(F.silu(c)) + return self.linear(self.norm_final(x) * scale) + + +class Ideogram4Transformer(nn.Module): + """A single Ideogram 4 backbone operating on a packed token sequence.""" + + def __init__(self, emb_dim, num_layers, num_heads, intermediate_size, adaln_dim, + in_channels, llm_features_dim, rope_theta, mrope_section, norm_eps, + dtype=None, device=None, operations=None): + super().__init__() + self.head_dim = emb_dim // num_heads + self.rope_theta = rope_theta + self.mrope_section = tuple(mrope_section) + + self.input_proj = operations.Linear(in_channels, emb_dim, bias=True, dtype=dtype, device=device) + self.llm_cond_norm = operations.RMSNorm(llm_features_dim, eps=1e-6, elementwise_affine=True, dtype=dtype, device=device) + self.llm_cond_proj = operations.Linear(llm_features_dim, emb_dim, bias=True, dtype=dtype, device=device) + self.t_embedding = Ideogram4EmbedScalar(emb_dim, input_range=(0.0, 1.0), dtype=dtype, device=device, operations=operations) + self.adaln_proj = operations.Linear(emb_dim, adaln_dim, bias=True, dtype=dtype, device=device) + + self.embed_image_indicator = operations.Embedding(2, emb_dim, dtype=dtype, device=device) + + self.layers = nn.ModuleList([ + Ideogram4TransformerBlock(emb_dim, intermediate_size, num_heads, norm_eps, adaln_dim, + dtype=dtype, device=device, operations=operations) + for _ in range(num_layers) + ]) + + self.final_layer = Ideogram4FinalLayer(emb_dim, in_channels, adaln_dim, dtype=dtype, device=device, operations=operations) + + def _backbone(self, llm_features, x, t, position_ids, attn_mask, indicator, transformer_options={}): + indicator = indicator.to(torch.long) + output_image_mask = (indicator == OUTPUT_IMAGE_INDICATOR).to(x.dtype).unsqueeze(-1) + + x = x * output_image_mask + h = self.input_proj(x) * output_image_mask + + t_cond = self.t_embedding(t, dtype=x.dtype) + if t.dim() == 1: + t_cond = t_cond.unsqueeze(1) + adaln_input = F.silu(self.adaln_proj(t_cond)) + + # h is zero on the text rows (content lives only on image rows), add writes the text features in place + if llm_features is not None: + L_text = llm_features.shape[1] + text_mask = (indicator[:, :L_text] == LLM_TOKEN_INDICATOR).to(x.dtype).unsqueeze(-1) + llm = self.llm_cond_norm(llm_features * text_mask) + llm = self.llm_cond_proj(llm) * text_mask + h[:, :L_text] = h[:, :L_text] + llm + + h = h + self.embed_image_indicator((indicator == OUTPUT_IMAGE_INDICATOR).to(torch.long), out_dtype=h.dtype) + + # Qwen3-VL interleaved MRoPE; position_ids (B, L, 3) -> (3, L) (same across batch). + freqs_cis = precompute_freqs_cis( + self.head_dim, position_ids[0].transpose(0, 1), self.rope_theta, + rope_dims=self.mrope_section, interleaved_mrope=True, device=position_ids.device, + ) + + if attn_mask is not None and attn_mask.dtype == torch.bool: + attn_mask = torch.zeros_like(attn_mask, dtype=h.dtype).masked_fill_(~attn_mask, -torch.finfo(h.dtype).max) + + for layer in self.layers: + h = layer(h, attn_mask, freqs_cis, adaln_input, transformer_options=transformer_options) + + return self.final_layer(h, adaln_input) + + +class Ideogram4Transformer2DModel(Ideogram4Transformer): + """Ideogram 4 single-stream DiT. + + Runs a packed ``[text, image]`` sequence when text context is supplied, or an image-only sequence when ``context is None``. + """ + + def __init__(self, image_model=None, in_channels=128, num_layers=34, num_attention_heads=18, attention_head_dim=256, intermediate_size=12288, + adaln_dim=512, llm_features_dim=53248, rope_theta=5000000, mrope_section=(24, 20, 20), norm_eps=1e-5, + dtype=None, device=None, operations=None, **kwargs): + emb_dim = num_attention_heads * attention_head_dim + super().__init__( + emb_dim=emb_dim, num_layers=num_layers, num_heads=num_attention_heads, + intermediate_size=intermediate_size, adaln_dim=adaln_dim, in_channels=in_channels, + llm_features_dim=llm_features_dim, rope_theta=rope_theta, mrope_section=mrope_section, + norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.dtype = dtype + self.in_channels = in_channels + self.out_channels = in_channels + # 128-dim token = patch (2x2) * ae_channels (32). + self.patch_size = 2 + self.ae_channels = in_channels // (self.patch_size * self.patch_size) + + def _img_to_tokens(self, x): + B, C, gh, gw = x.shape + x = x.view(B, self.ae_channels, self.patch_size, self.patch_size, gh, gw) + x = x.permute(0, 4, 5, 2, 3, 1) # (B, gh, gw, pi, pj, c) + return x.reshape(B, gh * gw, C) + + def _tokens_to_img(self, tokens, gh, gw): + B = tokens.shape[0] + C = tokens.shape[-1] + x = tokens.reshape(B, gh, gw, self.patch_size, self.patch_size, self.ae_channels) + x = x.permute(0, 5, 3, 4, 1, 2) # (B, c, pi, pj, gh, gw) + return x.reshape(B, C, gh, gw) + + def _image_position_ids(self, gh, gw, device): + h_idx = torch.arange(gh, device=device).view(-1, 1).expand(gh, gw).reshape(-1) + w_idx = torch.arange(gw, device=device).view(1, -1).expand(gh, gw).reshape(-1) + t_idx = torch.zeros_like(h_idx) + return torch.stack([t_idx, h_idx, w_idx], dim=1) + IMAGE_POSITION_OFFSET # (L_img, 3) + + def _run_conditional(self, x_chunk, context_chunk, attn_mask_chunk, t_chunk, gh, gw, transformer_options): + B = x_chunk.shape[0] + device = x_chunk.device + img_tokens = self._img_to_tokens(x_chunk) + L_img = img_tokens.shape[1] + L_text = context_chunk.shape[1] + L = L_text + L_img + latent_dim = img_tokens.shape[-1] + + x_full = torch.zeros(B, L, latent_dim, dtype=img_tokens.dtype, device=device) + x_full[:, L_text:] = img_tokens + + text_pos = torch.arange(L_text, device=device).view(-1, 1).expand(L_text, 3) + img_pos = self._image_position_ids(gh, gw, device) + position_ids = torch.cat([text_pos, img_pos], dim=0).unsqueeze(0).expand(B, L, 3) + + indicator = torch.empty(B, L, dtype=torch.long, device=device) + indicator[:, :L_text] = LLM_TOKEN_INDICATOR + indicator[:, L_text:] = OUTPUT_IMAGE_INDICATOR + + attn_mask = None + if attn_mask_chunk is not None: + segment_ids = torch.ones(B, L, dtype=torch.long, device=device) + pad = (attn_mask_chunk == 0) + segment_ids[:, :L_text][pad] = SEQUENCE_PADDING_INDICATOR + indicator[:, :L_text][pad] = 0 + # Block-diagonal mask from segment ids: (B, 1, L, L), True = attend. + attn_mask = (segment_ids.unsqueeze(2) == segment_ids.unsqueeze(1)).unsqueeze(1) + + out = self._backbone(context_chunk, x_full, t_chunk, position_ids, attn_mask, indicator, + transformer_options=transformer_options) + return self._tokens_to_img(out[:, L_text:], gh, gw) + + def _run_image_only(self, x_chunk, t_chunk, gh, gw, transformer_options): + B = x_chunk.shape[0] + device = x_chunk.device + img_tokens = self._img_to_tokens(x_chunk) + L_img = img_tokens.shape[1] + + position_ids = self._image_position_ids(gh, gw, device).unsqueeze(0).expand(B, L_img, 3) + indicator = torch.full((B, L_img), OUTPUT_IMAGE_INDICATOR, dtype=torch.long, device=device) + + # Image-only sequence is a single segment -> no mask, full attention, no LLM context. + out = self._backbone(None, img_tokens, t_chunk, position_ids, None, indicator, transformer_options=transformer_options) + return self._tokens_to_img(out, gh, gw) + + def forward(self, x, timesteps, context=None, attention_mask=None, transformer_options={}, **kwargs): + return comfy.patcher_extension.WrapperExecutor.new_class_executor( + self._forward, + self, + comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options), + ).execute(x, timesteps, context, attention_mask, transformer_options, **kwargs) + + def _forward(self, x, timesteps, context=None, attention_mask=None, transformer_options={}, **kwargs): + bs, c, gh, gw = x.shape + + timesteps = 1.0 - timesteps + + # unconditional pass + if context is None: + return -self._run_image_only(x, timesteps, gh, gw, transformer_options) + + return -self._run_conditional(x, context, attention_mask, timesteps, gh, gw, transformer_options) diff --git a/comfy/ldm/krea2/model.py b/comfy/ldm/krea2/model.py new file mode 100644 index 000000000..ecb16254f --- /dev/null +++ b/comfy/ldm/krea2/model.py @@ -0,0 +1,290 @@ +"""Krea 2 (K2) — single-stream MMDiT. + +Text tokens produced by a Qwen3-VL-4B 12-layer ``txtfusion`` adapter and patchified image tokens are +concatenated into one sequence and run through ``layers`` shared transformer blocks with +AdaLN-single modulation, GQA + per-head QK-norm + sigmoid-gated attention, SwiGLU MLP, and 3-axis RoPE. +""" + +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +import comfy.model_management +import comfy.patcher_extension +import comfy.ldm.common_dit +from comfy.ldm.flux.layers import EmbedND, timestep_embedding +from comfy.ldm.flux.math import apply_rope +from comfy.ldm.modules.attention import optimized_attention_masked + + +class RMSNorm(nn.Module): + """RMSNorm with the reference ``(1 + scale)`` weight convention (scale stored zero-centered).""" + + def __init__(self, features: int, eps: float = 1e-5, device=None, dtype=None, operations=None): + super().__init__() + self.eps = eps + self.scale = nn.Parameter(torch.empty(features, device=device, dtype=dtype)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + dtype = x.dtype + weight = comfy.model_management.cast_to(self.scale, dtype=torch.float32, device=x.device) + 1.0 + return F.rms_norm(x.float(), (x.shape[-1],), weight=weight, eps=self.eps).to(dtype) + + +class QKNorm(nn.Module): + def __init__(self, dim: int, device=None, dtype=None, operations=None): + super().__init__() + self.qnorm = RMSNorm(dim, device=device, dtype=dtype, operations=operations) + self.knorm = RMSNorm(dim, device=device, dtype=dtype, operations=operations) + + def forward(self, q, k): + return self.qnorm(q), self.knorm(k) + + +class SwiGLU(nn.Module): + def __init__(self, features: int, multiplier: int, bias: bool = False, multiple: int = 128, + device=None, dtype=None, operations=None): + super().__init__() + mlpdim = int(2 * features / 3) * multiplier + mlpdim = multiple * ((mlpdim + multiple - 1) // multiple) + self.gate = operations.Linear(features, mlpdim, bias=bias, device=device, dtype=dtype) + self.up = operations.Linear(features, mlpdim, bias=bias, device=device, dtype=dtype) + self.down = operations.Linear(mlpdim, features, bias=bias, device=device, dtype=dtype) + + def forward(self, x): + return self.down(F.silu(self.gate(x)).mul_(self.up(x))) + + +class Attention(nn.Module): + def __init__(self, dim: int, heads: int, kvheads: Optional[int] = None, bias: bool = False, + device=None, dtype=None, operations=None): + super().__init__() + self.heads = heads + self.kvheads = kvheads if kvheads is not None else heads + self.headdim = dim // self.heads + self.wq = operations.Linear(dim, self.headdim * self.heads, bias=bias, device=device, dtype=dtype) + self.wk = operations.Linear(dim, self.headdim * self.kvheads, bias=bias, device=device, dtype=dtype) + self.wv = operations.Linear(dim, self.headdim * self.kvheads, bias=bias, device=device, dtype=dtype) + self.gate = operations.Linear(dim, dim, bias=bias, device=device, dtype=dtype) + self.qknorm = QKNorm(self.headdim, device=device, dtype=dtype, operations=operations) + self.wo = operations.Linear(dim, dim, bias=bias, device=device, dtype=dtype) + + def forward(self, x, freqs=None, mask=None, transformer_options={}): + q, k, v, gate = self.wq(x), self.wk(x), self.wv(x), self.gate(x) + q = rearrange(q, "B L (H D) -> B H L D", H=self.heads) + k = rearrange(k, "B L (H D) -> B H L D", H=self.kvheads) + v = rearrange(v, "B L (H D) -> B H L D", H=self.kvheads) + q, k = self.qknorm(q, k) + if freqs is not None: + q, k = apply_rope(q, k, freqs) + if self.kvheads != self.heads: + rep = self.heads // self.kvheads + k = k.repeat_interleave(rep, dim=1) + v = v.repeat_interleave(rep, dim=1) + out = optimized_attention_masked(q, k, v, self.heads, mask=mask, skip_reshape=True, + transformer_options=transformer_options) + return self.wo(out * F.sigmoid(gate)) + + +class SimpleModulation(nn.Module): + def __init__(self, dim: int, device=None, dtype=None, operations=None): + super().__init__() + self.lin = nn.Parameter(torch.empty(2, dim, device=device, dtype=dtype)) + + def forward(self, vec): + out = vec + comfy.model_management.cast_to(self.lin, dtype=vec.dtype, device=vec.device).unsqueeze(0) + scale, shift = out.chunk(2, dim=1) + return scale, shift + + +class DoubleSharedModulation(nn.Module): + def __init__(self, dim: int, device=None, dtype=None, operations=None): + super().__init__() + self.lin = nn.Parameter(torch.empty(6 * dim, device=device, dtype=dtype)) + + def forward(self, vec): + out = vec + comfy.model_management.cast_to(self.lin, dtype=vec.dtype, device=vec.device) + return out.chunk(6, dim=-1) + + +class TextFusionBlock(nn.Module): + def __init__(self, features, heads, multiplier, bias=False, kvheads=None, device=None, dtype=None, operations=None): + super().__init__() + self.prenorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.postnorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.attn = Attention(features, heads, kvheads=kvheads, bias=bias, device=device, dtype=dtype, operations=operations) + self.mlp = SwiGLU(features, multiplier, bias, device=device, dtype=dtype, operations=operations) + + def forward(self, x, mask=None, transformer_options={}): + x = x + self.attn(self.prenorm(x), mask=mask, transformer_options=transformer_options) + x = x + self.mlp(self.postnorm(x)) + return x + + +class TextFusionTransformer(nn.Module): + def __init__(self, num_txt_layers, txt_dim, heads, multiplier, bias=False, kvheads=None, device=None, dtype=None, operations=None): + super().__init__() + self.layerwise_blocks = nn.ModuleList([ + TextFusionBlock(txt_dim, heads, multiplier, bias, kvheads, device=device, dtype=dtype, operations=operations) + for _ in range(2) + ]) + self.projector = operations.Linear(num_txt_layers, 1, bias=False, device=device, dtype=dtype) + self.refiner_blocks = nn.ModuleList([ + TextFusionBlock(txt_dim, heads, multiplier, bias, kvheads, device=device, dtype=dtype, operations=operations) + for _ in range(2) + ]) + + def forward(self, x, mask=None, transformer_options={}): + b, l, n, d = x.shape + x = x.reshape(b * l, n, d) + for block in self.layerwise_blocks: + x = block(x.contiguous(), mask=None, transformer_options=transformer_options) + x = rearrange(x, "(b l) n d -> b l d n", b=b, l=l) + x = self.projector(x).squeeze(-1) + for block in self.refiner_blocks: + x = block(x, mask=mask, transformer_options=transformer_options) + return x + + +class SingleStreamBlock(nn.Module): + def __init__(self, features, heads, multiplier, bias=False, kvheads=None, device=None, dtype=None, operations=None): + super().__init__() + self.mod = DoubleSharedModulation(features, device=device, dtype=dtype, operations=operations) + self.prenorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.postnorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.attn = Attention(features, heads, kvheads=kvheads, bias=bias, device=device, dtype=dtype, operations=operations) + self.mlp = SwiGLU(features, multiplier, bias, device=device, dtype=dtype, operations=operations) + + def forward(self, x, vec, freqs, mask=None, transformer_options={}): + prescale, preshift, pregate, postscale, postshift, postgate = self.mod(vec) + x = x + pregate * self.attn((1 + prescale) * self.prenorm(x) + preshift, freqs, mask, transformer_options=transformer_options) + x = x + postgate * self.mlp((1 + postscale) * self.postnorm(x) + postshift) + return x + + +class LastLayer(nn.Module): + def __init__(self, features, patch, channels, device=None, dtype=None, operations=None): + super().__init__() + self.norm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.linear = operations.Linear(features, patch * patch * channels, bias=True, device=device, dtype=dtype) + self.modulation = SimpleModulation(features, device=device, dtype=dtype, operations=operations) + + def forward(self, x, tvec): + scale, shift = self.modulation(tvec) + x = (1 + scale) * self.norm(x) + shift + return self.linear(x) + + +class SingleStreamDiT(nn.Module): + def __init__(self, features=6144, tdim=256, txtdim=2560, heads=48, kvheads=12, multiplier=4, + layers=28, patch=2, channels=16, bias=False, theta=1e3, txtlayers=12, + txtheads=20, txtkvheads=20, image_model=None, + device=None, dtype=None, operations=None, **kwargs): + super().__init__() + self.dtype = dtype + self.patch = patch + self.channels = channels + self.tdim = tdim + self.heads = heads + self.txtdim = txtdim + self.txtlayers = txtlayers + + headdim = features // heads + axes = [headdim - 12 * (headdim // 16), 6 * (headdim // 16), 6 * (headdim // 16)] + assert sum(axes) == headdim, f"axes {axes} sum != headdim {headdim}" + self.pe_embedder = EmbedND(dim=headdim, theta=int(theta), axes_dim=axes) + + self.first = operations.Linear(channels * patch ** 2, features, bias=True, device=device, dtype=dtype) + self.blocks = nn.ModuleList([ + SingleStreamBlock(features, heads, multiplier, bias, kvheads, device=device, dtype=dtype, operations=operations) + for _ in range(layers) + ]) + self.tmlp = nn.Sequential( + operations.Linear(tdim, features, device=device, dtype=dtype), + nn.GELU(approximate="tanh"), + operations.Linear(features, features, device=device, dtype=dtype), + ) + self.txtfusion = TextFusionTransformer(txtlayers, txtdim, txtheads, multiplier, bias, txtkvheads, + device=device, dtype=dtype, operations=operations) + self.txtmlp = nn.Sequential( + RMSNorm(txtdim, device=device, dtype=dtype, operations=operations), + operations.Linear(txtdim, features, device=device, dtype=dtype), + nn.GELU(approximate="tanh"), + operations.Linear(features, features, device=device, dtype=dtype), + ) + self.last = LastLayer(features, patch, channels, device=device, dtype=dtype, operations=operations) + self.tproj = nn.Sequential( + nn.GELU(approximate="tanh"), + operations.Linear(features, features * 6, device=device, dtype=dtype), + ) + + def forward(self, x, timesteps, context, attention_mask=None, transformer_options={}, **kwargs): + return comfy.patcher_extension.WrapperExecutor.new_class_executor( + self._forward, + self, + comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options), + ).execute(x, timesteps, context, attention_mask, transformer_options, **kwargs) + + def _forward(self, x, timesteps, context, attention_mask=None, transformer_options={}, **kwargs): + temporal = x.ndim == 5 + if temporal: + b5, c5, t5, h5, w5 = x.shape + x = x.reshape(b5 * t5, c5, h5, w5) + bs, c, H_orig, W_orig = x.shape + patch = self.patch + # Pad the latent up to a multiple of patch (as Flux/Lumina/QwenImage do); crop back at the end. + x = comfy.ldm.common_dit.pad_to_patch_size(x, (patch, patch)) + H, W = x.shape[-2], x.shape[-1] + h_, w_ = H // patch, W // patch + + # context arrives as (B, seq, txtlayers*txtdim); reshape to (B, txtlayers, seq, txtdim). + context = self._unpack_context(context) + + img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch, pw=patch) + img = self.first(img) + + t = self.tmlp(timestep_embedding(timesteps, self.tdim).unsqueeze(1).to(img.dtype)) + tvec = self.tproj(t) + + context = self.txtfusion(context, mask=None, transformer_options=transformer_options) + context = self.txtmlp(context) + + txtlen, imglen = context.shape[1], img.shape[1] + combined = torch.cat((context, img), dim=1) + + # Position ids: text at 0, image at (0, h_idx, w_idx). + device = combined.device + txtpos = torch.zeros(bs, txtlen, 3, device=device, dtype=torch.float32) + imgids = torch.zeros(h_, w_, 3, device=device, dtype=torch.float32) + imgids[..., 1] = torch.arange(h_, device=device, dtype=torch.float32)[:, None] + imgids[..., 2] = torch.arange(w_, device=device, dtype=torch.float32)[None, :] + imgpos = imgids.reshape(1, h_ * w_, 3).repeat(bs, 1, 1) + pos = torch.cat((txtpos, imgpos), dim=1) + + freqs = self.pe_embedder(pos) + + for block in self.blocks: + combined = block(combined, tvec, freqs, None, transformer_options=transformer_options) + + final = self.last(combined, t) + out = final[:, txtlen:txtlen + imglen, :] + out = rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", + h=h_, w=w_, ph=patch, pw=patch, c=self.channels) + out = out[:, :, :H_orig, :W_orig] # crop padding back off + if temporal: + out = out.reshape(b5, t5, self.channels, H_orig, W_orig).movedim(1, 2) + return out + + def _unpack_context(self, context): + # context: (B, seq, txtlayers*txtdim) -> (B, seq, txtlayers, txtdim). + b, seq, fused = context.shape + if fused != self.txtlayers * self.txtdim: + raise ValueError( + f"Krea2 expects conditioning with {self.txtlayers}x{self.txtdim}={self.txtlayers * self.txtdim} " + f"features (a {self.txtlayers}-layer Qwen3-VL stack) but got {fused}. " + f"Load the text encoder with CLIPLoader type 'krea2'." + ) + return context.reshape(b, seq, self.txtlayers, self.txtdim) diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py index e0a4a0f9b..9953b6679 100644 --- a/comfy/ldm/lightricks/model.py +++ b/comfy/ldm/lightricks/model.py @@ -1085,7 +1085,7 @@ class LTXVModel(LTXBaseModel): ) grid_mask = None - if keyframe_idxs is not None: + if keyframe_idxs is not None and keyframe_idxs.shape[2] > 0: additional_args.update({ "orig_patchified_shape": list(x.shape)}) denoise_mask = self.patchifier.patchify(denoise_mask)[0] grid_mask = ~torch.any(denoise_mask < 0, dim=-1)[0] @@ -1330,7 +1330,7 @@ class LTXVModel(LTXBaseModel): x = x * (1 + scale) + shift x = self.proj_out(x) - if keyframe_idxs is not None: + if keyframe_idxs is not None and keyframe_idxs.shape[2] > 0: grid_mask = kwargs["grid_mask"] orig_patchified_shape = kwargs["orig_patchified_shape"] full_x = torch.zeros(orig_patchified_shape, dtype=x.dtype, device=x.device) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 55360535a..e6500cff4 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -1,5 +1,6 @@ import math import sys +import inspect import torch import torch.nn.functional as F @@ -14,16 +15,16 @@ from .sub_quadratic_attention import efficient_dot_product_attention from comfy import model_management -TORCH_HAS_GQA = model_management.torch_version_numeric >= (2, 5) - if model_management.xformers_enabled(): import xformers import xformers.ops SAGE_ATTENTION_IS_AVAILABLE = False +SAGE_ATTENTION_SUPPORTS_MASK = False try: from sageattention import sageattn SAGE_ATTENTION_IS_AVAILABLE = True + SAGE_ATTENTION_SUPPORTS_MASK = "attn_mask" in inspect.signature(sageattn).parameters except ImportError as e: if model_management.sage_attention_enabled(): if e.name == "sageattention": @@ -89,6 +90,44 @@ def default(val, d): return val return d +def _gqa_repeat_factor(query_heads, key_heads, value_heads): + if key_heads != value_heads: + raise ValueError(f"Key/value head count mismatch for GQA: {key_heads} != {value_heads}") + if query_heads == key_heads: + return 1 + if query_heads % key_heads != 0: + raise ValueError(f"Query heads must be divisible by key/value heads for GQA: {query_heads} vs {key_heads}") + return query_heads // key_heads + +def _repeat_kv_for_gqa(k, v, query_heads, head_dim): + n_rep = _gqa_repeat_factor(query_heads, k.shape[head_dim], v.shape[head_dim]) + if n_rep > 1: + k = k.repeat_interleave(n_rep, dim=head_dim) + v = v.repeat_interleave(n_rep, dim=head_dim) + return k, v + +def _heads_from_dim(tensor, dim_head, name): + inner_dim = tensor.shape[-1] + if inner_dim % dim_head != 0: + raise ValueError(f"{name} inner dimension {inner_dim} is not divisible by head dimension {dim_head}") + return inner_dim // dim_head + +def _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, enable_gqa=False, expand_kv=True): + q = q.unsqueeze(3).reshape(b, -1, heads, dim_head) + if enable_gqa: + key_heads = _heads_from_dim(k, dim_head, "Key") + value_heads = _heads_from_dim(v, dim_head, "Value") + else: + key_heads = heads + value_heads = heads + k = k.unsqueeze(3).reshape(b, -1, key_heads, dim_head) + v = v.unsqueeze(3).reshape(b, -1, value_heads, dim_head) + if enable_gqa: + _gqa_repeat_factor(heads, key_heads, value_heads) + if expand_kv: + k, v = _repeat_kv_for_gqa(k, v, heads, -2) + return q, k, v + # feedforward class GEGLU(nn.Module): @@ -152,28 +191,19 @@ def attention_basic(q, k, v, heads, mask=None, attn_precision=None, skip_reshape b, _, dim_head = q.shape dim_head //= heads - if kwargs.get("enable_gqa", False) and q.shape[-3] != k.shape[-3]: - n_rep = q.shape[-3] // k.shape[-3] - k = k.repeat_interleave(n_rep, dim=-3) - v = v.repeat_interleave(n_rep, dim=-3) - scale = kwargs.get("scale", dim_head ** -0.5) h = heads if skip_reshape: - q, k, v = map( + if kwargs.get("enable_gqa", False): + k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + q, k, v = map( lambda t: t.reshape(b * heads, -1, dim_head), (q, k, v), ) else: - q, k, v = map( - lambda t: t.unsqueeze(3) - .reshape(b, -1, heads, dim_head) - .permute(0, 2, 1, 3) - .reshape(b * heads, -1, dim_head) - .contiguous(), - (q, k, v), - ) + q, k, v = _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, kwargs.get("enable_gqa", False)) + q, k, v = map(lambda t: t.permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head).contiguous(), (q, k, v)) # force cast to fp32 to avoid overflowing if attn_precision == torch.float32: @@ -231,13 +261,16 @@ def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None, query = query * (kwargs["scale"] * dim_head ** 0.5) if skip_reshape: + if kwargs.get("enable_gqa", False): + key, value = _repeat_kv_for_gqa(key, value, query.shape[-3], -3) query = query.reshape(b * heads, -1, dim_head) value = value.reshape(b * heads, -1, dim_head) key = key.reshape(b * heads, -1, dim_head).movedim(1, 2) else: - query = query.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head) - value = value.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head) - key = key.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 3, 1).reshape(b * heads, dim_head, -1) + query, key, value = _reshape_qkv_to_heads(query, key, value, b, heads, dim_head, kwargs.get("enable_gqa", False)) + query = query.permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head) + value = value.permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head) + key = key.permute(0, 2, 3, 1).reshape(b * heads, dim_head, -1) dtype = query.dtype @@ -304,19 +337,15 @@ def attention_split(q, k, v, heads, mask=None, attn_precision=None, skip_reshape scale = kwargs.get("scale", dim_head ** -0.5) if skip_reshape: - q, k, v = map( + if kwargs.get("enable_gqa", False): + k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + q, k, v = map( lambda t: t.reshape(b * heads, -1, dim_head), (q, k, v), ) else: - q, k, v = map( - lambda t: t.unsqueeze(3) - .reshape(b, -1, heads, dim_head) - .permute(0, 2, 1, 3) - .reshape(b * heads, -1, dim_head) - .contiguous(), - (q, k, v), - ) + q, k, v = _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, kwargs.get("enable_gqa", False)) + q, k, v = map(lambda t: t.permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head).contiguous(), (q, k, v)) r1 = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype) @@ -438,7 +467,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None, skip_resh disabled_xformers = True if disabled_xformers: - return attention_pytorch(q, k, v, heads, mask, skip_reshape=skip_reshape, **kwargs) + return attention_pytorch(q, k, v, heads, mask, skip_reshape=skip_reshape, skip_output_reshape=skip_output_reshape, **kwargs) if skip_reshape: # b h k d -> b k h d @@ -446,13 +475,12 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None, skip_resh lambda t: t.permute(0, 2, 1, 3), (q, k, v), ) + if kwargs.get("enable_gqa", False): + k, v = _repeat_kv_for_gqa(k, v, q.shape[-2], -2) # actually do the reshaping else: dim_head //= heads - q, k, v = map( - lambda t: t.reshape(b, -1, heads, dim_head), - (q, k, v), - ) + q, k, v = _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, kwargs.get("enable_gqa", False)) if mask is not None: # add a singleton batch dimension @@ -474,7 +502,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None, skip_resh mask = mask_out[..., :mask.shape[-1]] mask = mask.expand(b, heads, -1, -1) - out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=mask) + out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=mask, scale=kwargs.get("scale", None)) if skip_output_reshape: out = out.permute(0, 2, 1, 3) @@ -498,10 +526,8 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha else: b, _, dim_head = q.shape dim_head //= heads - q, k, v = map( - lambda t: t.view(b, -1, heads, dim_head).transpose(1, 2), - (q, k, v), - ) + q, k, v = _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, kwargs.get("enable_gqa", False), expand_kv=False) + q, k, v = map(lambda t: t.transpose(1, 2), (q, k, v)) if mask is not None: # add a batch dimension if there isn't already one @@ -511,9 +537,7 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha if mask.ndim == 3: mask = mask.unsqueeze(1) - # Pass through extra SDPA kwargs (scale, enable_gqa) if provided - # enable_gqa requires PyTorch 2.5+; older versions use manual KV expansion above - sdpa_keys = ("scale", "enable_gqa") if TORCH_HAS_GQA else ("scale",) + sdpa_keys = ("scale", "enable_gqa") sdpa_extra = {k: v for k, v in kwargs.items() if k in sdpa_keys} if SDP_BATCH_LIMIT >= b: @@ -541,20 +565,19 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha @wrap_attn def attention_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape=False, skip_output_reshape=False, **kwargs): - if kwargs.get("low_precision_attention", True) is False: + if kwargs.get("low_precision_attention", True) is False or (mask is not None and not SAGE_ATTENTION_SUPPORTS_MASK): return attention_pytorch(q, k, v, heads, mask=mask, skip_reshape=skip_reshape, skip_output_reshape=skip_output_reshape, **kwargs) exception_fallback = False if skip_reshape: b, _, _, dim_head = q.shape tensor_layout = "HND" + if kwargs.get("enable_gqa", False): + k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) else: b, _, dim_head = q.shape dim_head //= heads - q, k, v = map( - lambda t: t.view(b, -1, heads, dim_head), - (q, k, v), - ) + q, k, v = _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, kwargs.get("enable_gqa", False)) tensor_layout = "NHD" if mask is not None: @@ -565,8 +588,12 @@ def attention_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape= if mask.ndim == 3: mask = mask.unsqueeze(1) + sage_kwargs = {"is_causal": False, "tensor_layout": tensor_layout, "sm_scale": kwargs.get("scale", None), "smooth_k": False} + if mask is not None: + sage_kwargs["attn_mask"] = mask + try: - out = sageattn(q, k, v, attn_mask=mask, is_causal=False, tensor_layout=tensor_layout) + out = sageattn(q, k, v, **sage_kwargs) except Exception as e: logging.error("Error running sage attention: {}, using pytorch attention instead.".format(e)) exception_fallback = True @@ -616,7 +643,6 @@ def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape skip_output_reshape=skip_output_reshape, **kwargs ) - q_s, k_s, v_s = q, k, v N = q.shape[2] dim_head = D else: @@ -642,11 +668,15 @@ def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape **kwargs ) - if not skip_reshape: - q_s, k_s, v_s = map( - lambda t: t.view(B, -1, heads, dim_head).permute(0, 2, 1, 3).contiguous(), - (q, k, v), - ) + if skip_reshape: + q_s = q + if kwargs.get("enable_gqa", False): + k_s, v_s = _repeat_kv_for_gqa(k, v, H, -3) + else: + k_s, v_s = k, v + else: + q_s, k_s, v_s = _reshape_qkv_to_heads(q, k, v, B, heads, dim_head, kwargs.get("enable_gqa", False)) + q_s, k_s, v_s = map(lambda t: t.permute(0, 2, 1, 3).contiguous(), (q_s, k_s, v_s)) B, H, L, D = q_s.shape try: @@ -662,7 +692,7 @@ def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape q, k, v, heads, mask=mask, attn_precision=attn_precision, - skip_reshape=False, + skip_reshape=skip_reshape, skip_output_reshape=skip_output_reshape, **kwargs ) @@ -679,21 +709,22 @@ def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape return out try: - @torch.library.custom_op("flash_attention::flash_attn", mutates_args=()) + @torch.library.custom_op("comfy::flash_attn", mutates_args=()) def flash_attn_wrapper(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - dropout_p: float = 0.0, causal: bool = False) -> torch.Tensor: - return flash_attn_func(q, k, v, dropout_p=dropout_p, causal=causal) + dropout_p: float = 0.0, causal: bool = False, softmax_scale: float = -1.0) -> torch.Tensor: + softmax_scale_arg = None if softmax_scale == -1.0 else softmax_scale + return flash_attn_func(q, k, v, dropout_p=dropout_p, causal=causal, softmax_scale=softmax_scale_arg) @flash_attn_wrapper.register_fake - def flash_attn_fake(q, k, v, dropout_p=0.0, causal=False): + def flash_attn_fake(q, k, v, dropout_p=0.0, causal=False, softmax_scale=-1.0): # Output shape is the same as q return q.new_empty(q.shape) except AttributeError as error: FLASH_ATTN_ERROR = error def flash_attn_wrapper(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - dropout_p: float = 0.0, causal: bool = False) -> torch.Tensor: + dropout_p: float = 0.0, causal: bool = False, softmax_scale: float = -1.0) -> torch.Tensor: assert False, f"Could not define flash_attn_wrapper: {FLASH_ATTN_ERROR}" @wrap_attn @@ -703,10 +734,8 @@ def attention_flash(q, k, v, heads, mask=None, attn_precision=None, skip_reshape else: b, _, dim_head = q.shape dim_head //= heads - q, k, v = map( - lambda t: t.view(b, -1, heads, dim_head).transpose(1, 2), - (q, k, v), - ) + q, k, v = _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, kwargs.get("enable_gqa", False), expand_kv=False) + q, k, v = map(lambda t: t.transpose(1, 2), (q, k, v)) if mask is not None: # add a batch dimension if there isn't already one @@ -725,10 +754,16 @@ def attention_flash(q, k, v, heads, mask=None, attn_precision=None, skip_reshape v.transpose(1, 2), dropout_p=0.0, causal=False, + softmax_scale=kwargs.get("scale", -1.0), ).transpose(1, 2) except Exception as e: logging.warning(f"Flash Attention failed, using default SDPA: {e}") - out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False) + sdpa_extra = {} + if kwargs.get("enable_gqa", False): + sdpa_extra["enable_gqa"] = True + if "scale" in kwargs: + sdpa_extra["scale"] = kwargs["scale"] + out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False, **sdpa_extra) if not skip_output_reshape: out = ( out.transpose(1, 2).reshape(b, -1, heads * dim_head) @@ -1209,5 +1244,3 @@ class SpatialVideoTransformer(SpatialTransformer): x = self.proj_out(x) out = x + x_in return out - - diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py index fcbaa074f..e752d0ecb 100644 --- a/comfy/ldm/modules/diffusionmodules/model.py +++ b/comfy/ldm/modules/diffusionmodules/model.py @@ -22,7 +22,7 @@ def torch_cat_if_needed(xl, dim): else: return None -def get_timestep_embedding(timesteps, embedding_dim): +def get_timestep_embedding(timesteps, embedding_dim, flip_sin_to_cos=False, downscale_freq_shift=1): """ This matches the implementation in Denoising Diffusion Probabilistic Models: From Fairseq. @@ -33,11 +33,13 @@ def get_timestep_embedding(timesteps, embedding_dim): assert len(timesteps.shape) == 1 half_dim = embedding_dim // 2 - emb = math.log(10000) / (half_dim - 1) + emb = math.log(10000) / (half_dim - downscale_freq_shift) emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb) emb = emb.to(device=timesteps.device) emb = timesteps.float()[:, None] * emb[None, :] emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if flip_sin_to_cos: + emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1) if embedding_dim % 2 == 1: # zero pad emb = torch.nn.functional.pad(emb, (0,1,0,0)) return emb diff --git a/comfy/ldm/omnigen/omnigen2.py b/comfy/ldm/omnigen/omnigen2.py index 82edc92da..d18a9f6d0 100644 --- a/comfy/ldm/omnigen/omnigen2.py +++ b/comfy/ldm/omnigen/omnigen2.py @@ -8,6 +8,7 @@ import torch.nn.functional as F from einops import rearrange, repeat from comfy.ldm.lightricks.model import Timesteps from comfy.ldm.flux.layers import EmbedND +from comfy.ldm.flux.math import apply_rope1 from comfy.ldm.modules.attention import optimized_attention_masked import comfy.model_management import comfy.ldm.common_dit @@ -17,13 +18,11 @@ def apply_rotary_emb(x, freqs_cis): if x.shape[1] == 0: return x - t_ = x.reshape(*x.shape[:-1], -1, 1, 2) - t_out = freqs_cis[..., 0] * t_[..., 0] + freqs_cis[..., 1] * t_[..., 1] - return t_out.reshape(*x.shape).to(dtype=x.dtype) + return apply_rope1(x, freqs_cis) def swiglu(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - return F.silu(x) * y + return F.silu(x, inplace=True).mul_(y) class TimestepEmbedding(nn.Module): @@ -142,11 +141,8 @@ class Attention(nn.Module): key = key.transpose(1, 2) value = value.transpose(1, 2) - if self.kv_heads < self.heads: - key = key.repeat_interleave(self.heads // self.kv_heads, dim=1) - value = value.repeat_interleave(self.heads // self.kv_heads, dim=1) - - hidden_states = optimized_attention_masked(query, key, value, self.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options) + gqa_kwargs = {"enable_gqa": True} if self.kv_heads < self.heads else {} + hidden_states = optimized_attention_masked(query, key, value, self.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options, **gqa_kwargs) hidden_states = self.to_out[0](hidden_states) return hidden_states diff --git a/comfy/ldm/pixeldit/model.py b/comfy/ldm/pixeldit/model.py index b044b9b29..3b30b9226 100644 --- a/comfy/ldm/pixeldit/model.py +++ b/comfy/ldm/pixeldit/model.py @@ -197,6 +197,9 @@ class PixDiT_T2I(nn.Module): """Hook for subclasses to inject per-block state into the patch stream (e.g. PiD's LQ gate).""" return s + def _pre_pixel_blocks(self, s, **kwargs): + return s + def _forward(self, x, timesteps, context=None, attention_mask=None, transformer_options={}, **kwargs): H_orig, W_orig = x.shape[2], x.shape[3] x = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size)) @@ -226,6 +229,7 @@ class PixDiT_T2I(nn.Module): s, y_emb = blk(s, y_emb, condition, pos_img, pos_txt, None, transformer_options=transformer_options) s = F.silu(t_emb + s) + s = self._pre_pixel_blocks(s, **kwargs) s_cond = s.view(B * L, self.hidden_size) x_pixels = self.pixel_embedder(x, patch_size=self.patch_size) for blk in self.pixel_blocks: diff --git a/comfy/ldm/pixeldit/pid.py b/comfy/ldm/pixeldit/pid.py index 21b73907a..8590408d9 100644 --- a/comfy/ldm/pixeldit/pid.py +++ b/comfy/ldm/pixeldit/pid.py @@ -13,15 +13,15 @@ from .model import PixDiT_T2I from .modules import precompute_freqs_cis_2d -class SigmaAwareGatePerTokenPerDim(nn.Module): +class SigmaAwareGate(nn.Module): """gate = sigmoid(content_proj(cat[x, lq]) - exp(log_alpha) * sigma); out = x + gate * lq. Trained init gives ~0.88 gate at sigma=0, ~0.05 at sigma=1. """ - def __init__(self, dim: int, dtype=None, device=None, operations=None): + def __init__(self, dim: int, per_token: bool = False, dtype=None, device=None, operations=None): super().__init__() - self.content_proj = operations.Linear(dim * 2, dim, dtype=dtype, device=device) + self.content_proj = operations.Linear(dim * 2, 1 if per_token else dim, dtype=dtype, device=device) self.log_alpha = nn.Parameter(torch.empty((), dtype=dtype, device=device)) def forward(self, x: torch.Tensor, lq: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: @@ -36,15 +36,15 @@ class SigmaAwareGatePerTokenPerDim(nn.Module): class ResBlock(nn.Module): """Pre-activation ResNet block: GN -> SiLU -> Conv -> GN -> SiLU -> Conv + skip.""" - def __init__(self, channels: int, num_groups: int = 4, dtype=None, device=None, operations=None): + def __init__(self, channels: int, num_groups: int = 4, conv_padding_mode: str = "zeros", dtype=None, device=None, operations=None): super().__init__() self.block = nn.Sequential( operations.GroupNorm(num_groups, channels, dtype=dtype, device=device), nn.SiLU(), - operations.Conv2d(channels, channels, kernel_size=3, padding=1, dtype=dtype, device=device), + operations.Conv2d(channels, channels, kernel_size=3, padding=1, padding_mode=conv_padding_mode, dtype=dtype, device=device), operations.GroupNorm(num_groups, channels, dtype=dtype, device=device), nn.SiLU(), - operations.Conv2d(channels, channels, kernel_size=3, padding=1, dtype=dtype, device=device), + operations.Conv2d(channels, channels, kernel_size=3, padding=1, padding_mode=conv_padding_mode, dtype=dtype, device=device), ) def forward(self, x: torch.Tensor) -> torch.Tensor: @@ -62,9 +62,13 @@ class LQProjection2D(nn.Module): patch_size: int = 16, sr_scale: int = 4, latent_spatial_down_factor: int = 8, + latent_unpatchify_factor: int = 1, num_res_blocks: int = 4, num_outputs: int = 7, interval: int = 2, + conv_padding_mode: str = "zeros", + gate_per_token: bool = False, + pit_output: bool = False, dtype=None, device=None, operations=None, ): super().__init__() @@ -74,34 +78,38 @@ class LQProjection2D(nn.Module): self.patch_size = patch_size self.sr_scale = sr_scale self.latent_spatial_down_factor = latent_spatial_down_factor + self.latent_unpatchify_factor = latent_unpatchify_factor self.num_outputs = num_outputs self.interval = interval - z_to_patch_ratio = (sr_scale * latent_spatial_down_factor) / patch_size + effective_latent_channels = latent_channels // (latent_unpatchify_factor * latent_unpatchify_factor) + effective_spatial_down_factor = latent_spatial_down_factor // latent_unpatchify_factor + z_to_patch_ratio = (sr_scale * effective_spatial_down_factor) / patch_size self.z_to_patch_ratio = z_to_patch_ratio if z_to_patch_ratio >= 1: self.latent_fold_factor = 0 - latent_proj_in_ch = latent_channels + latent_proj_in_ch = effective_latent_channels else: fold_factor = int(1 / z_to_patch_ratio) assert fold_factor * z_to_patch_ratio == 1.0 self.latent_fold_factor = fold_factor - latent_proj_in_ch = latent_channels * fold_factor * fold_factor + latent_proj_in_ch = effective_latent_channels * fold_factor * fold_factor layers = [ - operations.Conv2d(latent_proj_in_ch, hidden_dim, kernel_size=3, padding=1, dtype=dtype, device=device), + operations.Conv2d(latent_proj_in_ch, hidden_dim, kernel_size=3, padding=1, padding_mode=conv_padding_mode, dtype=dtype, device=device), nn.SiLU(), - operations.Conv2d(hidden_dim, hidden_dim, kernel_size=3, padding=1, dtype=dtype, device=device), + operations.Conv2d(hidden_dim, hidden_dim, kernel_size=3, padding=1, padding_mode=conv_padding_mode, dtype=dtype, device=device), ] for _ in range(num_res_blocks): - layers.append(ResBlock(hidden_dim, dtype=dtype, device=device, operations=operations)) + layers.append(ResBlock(hidden_dim, conv_padding_mode=conv_padding_mode, dtype=dtype, device=device, operations=operations)) self.latent_proj = nn.Sequential(*layers) self.output_heads = nn.ModuleList( [operations.Linear(hidden_dim, out_dim, dtype=dtype, device=device) for _ in range(num_outputs)] ) + self.pit_head = operations.Linear(hidden_dim, out_dim, dtype=dtype, device=device) if pit_output else None self.gate_modules = nn.ModuleList( - [SigmaAwareGatePerTokenPerDim(out_dim, dtype=dtype, device=device, operations=operations) + [SigmaAwareGate(out_dim, per_token=gate_per_token, dtype=dtype, device=device, operations=operations) for _ in range(num_outputs)] ) @@ -115,6 +123,11 @@ class LQProjection2D(nn.Module): return self.gate_modules[out_idx](x, lq_feature, sigma) def _align_latent_to_patch_grid(self, lq_latent: torch.Tensor, pH: int, pW: int) -> torch.Tensor: + f = self.latent_unpatchify_factor + if f > 1: + B, C, H, W = lq_latent.shape + lq_latent = lq_latent.reshape(B, C // (f * f), f, f, H, W) + lq_latent = lq_latent.permute(0, 1, 4, 2, 5, 3).reshape(B, C // (f * f), H * f, W * f) B, z_dim = lq_latent.shape[:2] if self.z_to_patch_ratio >= 1: if lq_latent.shape[2] != pH or lq_latent.shape[3] != pW: @@ -134,7 +147,10 @@ class LQProjection2D(nn.Module): feat = self._align_latent_to_patch_grid(lq_latent, target_pH, target_pW) B, C, H, W = feat.shape tokens = feat.permute(0, 2, 3, 1).contiguous().view(B, H * W, C) - return [head(tokens) for head in self.output_heads] + outputs = [head(tokens) for head in self.output_heads] + if self.pit_head is not None: + outputs.append(self.pit_head(tokens)) + return outputs class PidNet(PixDiT_T2I): @@ -148,6 +164,10 @@ class PidNet(PixDiT_T2I): lq_interval: int = 2, sr_scale: int = 4, latent_spatial_down_factor: int = 8, + lq_latent_unpatchify_factor: int = 1, + lq_conv_padding_mode: str = "zeros", + lq_gate_per_token: bool = False, + pit_lq_inject: bool = False, rope_ref_h: int = 1024, # NTK ref resolution in PIXEL units: 1024px / patch=16 -> grid_ref=64. rope_ref_w: int = 1024, image_model=None, @@ -165,6 +185,8 @@ class PidNet(PixDiT_T2I): for blk in self.pixel_blocks: blk._rope_fn = _pit_rope_fn + self.pit_lq_inject = pit_lq_inject + num_lq_outputs = (self.patch_depth + lq_interval - 1) // lq_interval self.lq_proj = LQProjection2D( latent_channels=lq_latent_channels, @@ -173,13 +195,20 @@ class PidNet(PixDiT_T2I): patch_size=self.patch_size, sr_scale=sr_scale, latent_spatial_down_factor=latent_spatial_down_factor, + latent_unpatchify_factor=lq_latent_unpatchify_factor, num_res_blocks=lq_num_res_blocks, num_outputs=num_lq_outputs, interval=lq_interval, + conv_padding_mode=lq_conv_padding_mode, + gate_per_token=lq_gate_per_token, + pit_output=pit_lq_inject, dtype=dtype, device=device, operations=operations, ) + self.pit_lq_gate = SigmaAwareGate( + self.hidden_size, per_token=lq_gate_per_token, dtype=dtype, device=device, operations=operations + ) if pit_lq_inject else None def _fetch_patch_pos(self, height, width, device, dtype, **rope_opts): return precompute_freqs_cis_2d( @@ -197,6 +226,11 @@ class PidNet(PixDiT_T2I): return s return self.lq_proj.gate(s, pid_lq_features[out_idx], pid_degrade_sigma, out_idx) + def _pre_pixel_blocks(self, s, pid_pit_lq_feature=None, pid_degrade_sigma=None, **kwargs): + if pid_pit_lq_feature is None: + return s + return self.pit_lq_gate(s, pid_pit_lq_feature, pid_degrade_sigma) + def _forward(self, x, timesteps, context=None, attention_mask=None, transformer_options={}, lq_latent=None, degrade_sigma=None, **kwargs): if lq_latent is None: raise ValueError("PidNet requires lq_latent — attach via PiDConditioning") @@ -216,12 +250,14 @@ class PidNet(PixDiT_T2I): degrade_sigma = degrade_sigma.expand(B).contiguous() lq_features = self.lq_proj(lq_latent=lq_latent.to(x), target_pH=Hs, target_pW=Ws) + pit_lq_feature = lq_features.pop() if self.pit_lq_inject else None return super()._forward( x, timesteps, context=context, attention_mask=attention_mask, transformer_options=transformer_options, pid_lq_features=lq_features, + pid_pit_lq_feature=pit_lq_feature, pid_degrade_sigma=degrade_sigma, **kwargs, ) diff --git a/comfy/ldm/qwen_image/model.py b/comfy/ldm/qwen_image/model.py index 3462d8108..e49886dd9 100644 --- a/comfy/ldm/qwen_image/model.py +++ b/comfy/ldm/qwen_image/model.py @@ -51,6 +51,18 @@ class FeedForward(nn.Module): return hidden_states +# Addin this back because Nunchaku custom nodes rely on it, see comment here: +# https://github.com/Comfy-Org/ComfyUI/pull/14178#issuecomment-4640475161 +# TODO: Eventually remove this once we natively support SVDQuants +def apply_rotary_emb(x, freqs_cis): + if x.shape[1] == 0: + return x + + t_ = x.reshape(*x.shape[:-1], -1, 1, 2) + t_out = freqs_cis[..., 0] * t_[..., 0] + freqs_cis[..., 1] * t_[..., 1] + return t_out.reshape(*x.shape) + + class QwenTimestepProjEmbeddings(nn.Module): def __init__(self, embedding_dim, pooled_projection_dim, use_additional_t_cond=False, dtype=None, device=None, operations=None): super().__init__() diff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.py new file mode 100644 index 000000000..11b4c1e4a --- /dev/null +++ b/comfy/ldm/seedvr/attention.py @@ -0,0 +1,51 @@ +import torch + +from comfy.ldm.modules import attention as _attention + + +def _var_attention_qkv(q, k, v, heads, skip_reshape): + if skip_reshape: + return q, k, v, q.shape[-1] + total_tokens, embed_dim = q.shape + head_dim = embed_dim // heads + return ( + q.view(total_tokens, heads, head_dim), + k.view(k.shape[0], heads, head_dim), + v.view(v.shape[0], heads, head_dim), + head_dim, + ) + + +def _var_attention_output(out, heads, head_dim, skip_output_reshape): + if skip_output_reshape: + return out + return out.reshape(-1, heads * head_dim) + + +def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs): + q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape) + + q_split_indices = cu_seqlens_q[1:-1] + k_split_indices = cu_seqlens_k[1:-1] + if k.shape[0] != v.shape[0]: + raise ValueError("cu_seqlens_k does not match v token count") + + q_splits = torch.tensor_split(q, q_split_indices, dim=0) + k_splits = torch.tensor_split(k, k_split_indices, dim=0) + v_splits = torch.tensor_split(v, k_split_indices, dim=0) + if len(q_splits) != len(k_splits) or len(q_splits) != len(v_splits): + raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same sequence count") + + out = [] + for q_i, k_i, v_i in zip(q_splits, k_splits, v_splits): + q_i = q_i.permute(1, 0, 2).unsqueeze(0) + k_i = k_i.permute(1, 0, 2).unsqueeze(0) + v_i = v_i.permute(1, 0, 2).unsqueeze(0) + out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True) + out.append(out_i.squeeze(0).permute(1, 0, 2)) + + out = torch.cat(out, dim=0) + return _var_attention_output(out, heads, head_dim, skip_output_reshape) + + +optimized_var_attention = var_attention_optimized_split diff --git a/comfy/ldm/seedvr/color_fix.py b/comfy/ldm/seedvr/color_fix.py new file mode 100644 index 000000000..a43cb5270 --- /dev/null +++ b/comfy/ldm/seedvr/color_fix.py @@ -0,0 +1,301 @@ +import torch +import torch.nn.functional as F +from torch import Tensor + +from comfy.ldm.seedvr.constants import ( + CIELAB_DELTA, + CIELAB_KAPPA, + D65_WHITE_X, + D65_WHITE_Z, + WAVELET_DECOMP_LEVELS, +) + + +def wavelet_blur(image: Tensor, radius): + max_safe_radius = max(1, min(image.shape[-2:]) // 8) + if radius > max_safe_radius: + radius = max_safe_radius + + num_channels = image.shape[1] + + kernel_vals = [ + [0.0625, 0.125, 0.0625], + [0.125, 0.25, 0.125], + [0.0625, 0.125, 0.0625], + ] + kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device) + kernel = kernel[None, None].repeat(num_channels, 1, 1, 1) + + image = F.pad(image, (radius, radius, radius, radius), mode='replicate') + output = F.conv2d(image, kernel, groups=num_channels, dilation=radius) + + return output + +def wavelet_decomposition(image: Tensor, levels: int = WAVELET_DECOMP_LEVELS): + high_freq = torch.zeros_like(image) + + for i in range(levels): + radius = 2 ** i + low_freq = wavelet_blur(image, radius) + high_freq.add_(image).sub_(low_freq) + image = low_freq + + return high_freq, low_freq + +def wavelet_reconstruction(content_feat: Tensor, style_feat: Tensor) -> Tensor: + + if content_feat.shape != style_feat.shape: + if len(content_feat.shape) >= 3: + style_feat = F.interpolate( + style_feat, + size=content_feat.shape[-2:], + mode='bilinear', + align_corners=False + ) + + content_high_freq, content_low_freq = wavelet_decomposition(content_feat) + del content_low_freq + + style_high_freq, style_low_freq = wavelet_decomposition(style_feat) + del style_high_freq + + if content_high_freq.shape != style_low_freq.shape: + style_low_freq = F.interpolate( + style_low_freq, + size=content_high_freq.shape[-2:], + mode='bilinear', + align_corners=False + ) + + content_high_freq.add_(style_low_freq) + + return content_high_freq.clamp_(-1.0, 1.0) + +def _histogram_matching_channel(source: Tensor, reference: Tensor) -> Tensor: + original_shape = source.shape + + source_flat = source.flatten() + reference_flat = reference.flatten() + + source_sorted, source_indices = torch.sort(source_flat) + reference_sorted, _ = torch.sort(reference_flat) + del reference_flat + + n_source = len(source_sorted) + n_reference = len(reference_sorted) + + if n_source == n_reference: + matched_sorted = reference_sorted + else: + source_quantiles = torch.linspace(0, 1, n_source, device=source.device) + ref_indices = (source_quantiles * (n_reference - 1)).long() + ref_indices.clamp_(0, n_reference - 1) + matched_sorted = reference_sorted[ref_indices] + del source_quantiles, ref_indices, reference_sorted + + del source_sorted, source_flat + + inverse_indices = torch.argsort(source_indices) + del source_indices + matched_flat = matched_sorted[inverse_indices] + del matched_sorted, inverse_indices + + return matched_flat.reshape(original_shape) + +def _lab_to_rgb_batch(lab: Tensor, matrix_inv: Tensor, epsilon: float, kappa: float) -> Tensor: + L, a, b = lab[:, 0], lab[:, 1], lab[:, 2] + + fy = (L + 16.0) / 116.0 + fx = a.div(500.0).add_(fy) + fz = fy - b / 200.0 + del L, a, b + + x = torch.where( + fx > epsilon, + torch.pow(fx, 3.0), + fx.mul(116.0).sub_(16.0).div_(kappa) + ) + y = torch.where( + fy > epsilon, + torch.pow(fy, 3.0), + fy.mul(116.0).sub_(16.0).div_(kappa) + ) + z = torch.where( + fz > epsilon, + torch.pow(fz, 3.0), + fz.mul(116.0).sub_(16.0).div_(kappa) + ) + del fx, fy, fz + + x.mul_(D65_WHITE_X) + z.mul_(D65_WHITE_Z) + + xyz = torch.stack([x, y, z], dim=1) + del x, y, z + + B, _, H, W = xyz.shape + xyz_flat = xyz.permute(0, 2, 3, 1).reshape(-1, 3) + del xyz + + xyz_flat = xyz_flat.to(dtype=matrix_inv.dtype) + rgb_linear_flat = torch.matmul(xyz_flat, matrix_inv.T) + del xyz_flat + + rgb_linear = rgb_linear_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2) + del rgb_linear_flat + + mask = rgb_linear > 0.0031308 + rgb = torch.where( + mask, + torch.pow(torch.clamp(rgb_linear, min=0.0), 1.0 / 2.4).mul_(1.055).sub_(0.055), + rgb_linear * 12.92 + ) + del mask, rgb_linear + + return torch.clamp(rgb, 0.0, 1.0) + +def _rgb_to_lab_batch(rgb: Tensor, matrix: Tensor, epsilon: float, kappa: float) -> Tensor: + mask = rgb > 0.04045 + rgb_linear = torch.where( + mask, + torch.pow((rgb + 0.055) / 1.055, 2.4), + rgb / 12.92 + ) + del mask + + B, _, H, W = rgb_linear.shape + rgb_flat = rgb_linear.permute(0, 2, 3, 1).reshape(-1, 3) + del rgb_linear + + rgb_flat = rgb_flat.to(dtype=matrix.dtype) + xyz_flat = torch.matmul(rgb_flat, matrix.T) + del rgb_flat + + xyz = xyz_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2) + del xyz_flat + + xyz[:, 0].div_(D65_WHITE_X) + xyz[:, 2].div_(D65_WHITE_Z) + + epsilon_cubed = epsilon ** 3 + mask = xyz > epsilon_cubed + f_xyz = torch.where( + mask, + torch.pow(xyz, 1.0 / 3.0), + xyz.mul(kappa).add_(16.0).div_(116.0) + ) + del xyz, mask + + L = f_xyz[:, 1].mul(116.0).sub_(16.0) + a = (f_xyz[:, 0] - f_xyz[:, 1]).mul_(500.0) + b = (f_xyz[:, 1] - f_xyz[:, 2]).mul_(200.0) + del f_xyz + + return torch.stack([L, a, b], dim=1) + +def lab_color_transfer( + content_feat: Tensor, + style_feat: Tensor, + luminance_weight: float = 0.8 +) -> Tensor: + content_feat = wavelet_reconstruction(content_feat, style_feat) + + if content_feat.shape != style_feat.shape: + style_feat = F.interpolate( + style_feat, + size=content_feat.shape[-2:], + mode='bilinear', + align_corners=False + ) + + device = content_feat.device + original_dtype = content_feat.dtype + content_feat = content_feat.float() + style_feat = style_feat.float() + + rgb_to_xyz_matrix = torch.tensor([ + [0.4124564, 0.3575761, 0.1804375], + [0.2126729, 0.7151522, 0.0721750], + [0.0193339, 0.1191920, 0.9503041] + ], dtype=torch.float32, device=device) + + xyz_to_rgb_matrix = torch.tensor([ + [ 3.2404542, -1.5371385, -0.4985314], + [-0.9692660, 1.8760108, 0.0415560], + [ 0.0556434, -0.2040259, 1.0572252] + ], dtype=torch.float32, device=device) + + epsilon = CIELAB_DELTA + kappa = CIELAB_KAPPA + + content_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0) + style_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0) + + content_lab = _rgb_to_lab_batch(content_feat, rgb_to_xyz_matrix, epsilon, kappa) + del content_feat + + style_lab = _rgb_to_lab_batch(style_feat, rgb_to_xyz_matrix, epsilon, kappa) + del style_feat, rgb_to_xyz_matrix + + matched_a = _histogram_matching_channel(content_lab[:, 1], style_lab[:, 1]) + matched_b = _histogram_matching_channel(content_lab[:, 2], style_lab[:, 2]) + + if luminance_weight < 1.0: + matched_L = _histogram_matching_channel(content_lab[:, 0], style_lab[:, 0]) + result_L = content_lab[:, 0].mul(luminance_weight).add_(matched_L.mul(1.0 - luminance_weight)) + del matched_L + else: + result_L = content_lab[:, 0] + + del content_lab, style_lab + + result_lab = torch.stack([result_L, matched_a, matched_b], dim=1) + del result_L, matched_a, matched_b + + result_rgb = _lab_to_rgb_batch(result_lab, xyz_to_rgb_matrix, epsilon, kappa) + del result_lab, xyz_to_rgb_matrix + + result = result_rgb.mul_(2.0).sub_(1.0) + del result_rgb + + result = result.to(original_dtype) + + return result + + +def wavelet_color_transfer(content_feat: Tensor, style_feat: Tensor) -> Tensor: + return wavelet_reconstruction(content_feat, style_feat) + + +def adain_color_transfer(content_feat: Tensor, style_feat: Tensor, eps: float = 1e-5) -> Tensor: + if content_feat.shape != style_feat.shape: + style_feat = F.interpolate( + style_feat, + size=content_feat.shape[-2:], + mode='bilinear', + align_corners=False, + ) + + original_dtype = content_feat.dtype + content_feat = content_feat.float() + style_feat = style_feat.float() + + b, c = content_feat.shape[:2] + content_flat = content_feat.reshape(b, c, -1) + style_flat = style_feat.reshape(b, c, -1) + + content_mean = content_flat.mean(dim=2).reshape(b, c, 1, 1) + content_std = (content_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1) + style_mean = style_flat.mean(dim=2).reshape(b, c, 1, 1) + style_std = (style_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1) + del content_flat, style_flat + + normalized = (content_feat - content_mean) / content_std + del content_mean, content_std + result = normalized * style_std + style_mean + del normalized, style_mean, style_std + + result = result.clamp_(-1.0, 1.0) + if result.dtype != original_dtype: + result = result.to(original_dtype) + return result diff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.py new file mode 100644 index 000000000..12c4b4bef --- /dev/null +++ b/comfy/ldm/seedvr/constants.py @@ -0,0 +1,48 @@ +"""SeedVR2 constants.""" + +# Temporal chunk-size law: the sampler's activation wall is linear in +# T_latent * pixel area (17-cell resolution sweep + T bisection, RTX 5090, 3b fp16): +# max_latent_frames = (free_GiB - RESERVED - K*SIGMA) / (GIB_PER_MPX_FRAME * megapixels) +# RESERVED covers model staging plus fixed CUDA/torch overhead; SIGMA is the measured +# run-to-run spread of the wall; K=4 trades ~10% smaller chunks for ~1e-5 OOM odds. +SEEDVR2_CHUNK_GIB_PER_MPX_FRAME = 0.55 +SEEDVR2_CHUNK_RESERVED_GIB = 8.5 +SEEDVR2_CHUNK_SIGMA_GIB = 0.55 +SEEDVR2_CHUNK_SIGMA_K = 4 + +SEEDVR2_7B_VID_DIM = 3072 +SEEDVR2_OOM_BACKOFF_DIVISOR = 2 +SEEDVR2_DTYPE_BYTES_FLOOR = 4 +SEEDVR2_7B_MLP_CHUNK = 8192 +SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS = 4096 # partial-RoPE application token-chunk. +SEEDVR2_LATENT_CHANNELS = 16 + +SEEDVR2_COLOR_MEM_HEADROOM = 0.75 +SEEDVR2_LAB_SCALE_MULTIPLIER = 13 +SEEDVR2_WAVELET_SCALE_MULTIPLIER = 10 # per-frame byte multiplier, wavelet path. +SEEDVR2_ADAIN_SCALE_MULTIPLIER = 6 + +BYTEDANCE_VAE_SCALING_FACTOR = 0.9152 # configs_3b/main.yaml:57. +BYTEDANCE_VAE_SHIFTING_FACTOR = 0.0 +BYTEDANCE_VAE_CONV_MEM_GIB = 0.5 +BYTEDANCE_VAE_NORM_MEM_GIB = 0.5 +BYTEDANCE_LOGVAR_CLAMP_MIN = -30.0 # video_vae_v3/modules/types.py:28. +BYTEDANCE_LOGVAR_CLAMP_MAX = 20.0 # video_vae_v3/modules/types.py:28. +BYTEDANCE_GN_CHUNKS_FP16 = 4 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp16). +BYTEDANCE_GN_CHUNKS_FP32 = 2 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp32). +BYTEDANCE_BLOCK_OUT_CHANNELS = (128, 256, 512, 512) # s8_c16_t4_inflation_sd3.yaml:7-11. +BYTEDANCE_SLICING_SAMPLE_MIN = 4 # s8_c16_t4_inflation_sd3.yaml:22 (slicing_sample_min_size). +BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE = 4 # infer.py:230 (temporal_downsample_factor); the 4n+1 factor. +BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE = 8 # infer.py:231 (spatial_downsample_factor). +BYTEDANCE_720P_REF_AREA = 45 * 80 # dit_v2/window.py:32 (720p reference area for window scaling). +BYTEDANCE_MAX_TEMPORAL_WINDOW = 30 # dit_v2/window.py:35 (max temporal window frames). +BYTEDANCE_ROPE_MAX_FREQ = 256 # dit_v2/rope.py:31 (pixel-RoPE max frequency). +BYTEDANCE_SINUSOIDAL_DIM = 256 # dit_3b/nadit.py:120 (timestep sinusoidal embed dim). + +ROPE_THETA = 10000 # RoPE base; Su et al., "RoFormer", arXiv:2104.09864. + +CIELAB_DELTA = 6.0 / 29.0 # CIE 15 (delta). +CIELAB_KAPPA = (29.0 / 3.0) ** 3 # CIE 15 (kappa). +D65_WHITE_X = 0.95047 # CIE D65 standard illuminant Xn (Yn = 1). +D65_WHITE_Z = 1.08883 # CIE D65 standard illuminant Zn. +WAVELET_DECOMP_LEVELS = 5 # wavelet color-fix decomposition depth (GIMP/Krita; StableSR). diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.py new file mode 100644 index 000000000..a978698d5 --- /dev/null +++ b/comfy/ldm/seedvr/model.py @@ -0,0 +1,1361 @@ +from dataclasses import dataclass +from typing import Optional, Tuple, Union, List, Dict, Any, Callable +import torch.nn.functional as F +from math import ceil, pi +import torch +from itertools import accumulate, chain +from comfy.ldm.modules.diffusionmodules.model import get_timestep_embedding +from comfy.ldm.seedvr.attention import optimized_var_attention +from torch.nn.modules.utils import _triple +from torch import nn +import math +from comfy.ldm.flux.math import apply_rope1 +from comfy.ldm.seedvr.constants import ( + BYTEDANCE_720P_REF_AREA, + BYTEDANCE_MAX_TEMPORAL_WINDOW, + BYTEDANCE_ROPE_MAX_FREQ, + BYTEDANCE_SINUSOIDAL_DIM, + ROPE_THETA, + SEEDVR2_7B_MLP_CHUNK, + SEEDVR2_7B_VID_DIM, + SEEDVR2_LATENT_CHANNELS, + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, +) +import comfy.model_management +import comfy.ops + +class Cache: + def __init__(self, disable=False, prefix="", cache=None): + self.cache = cache if cache is not None else {} + self.disable = disable + self.prefix = prefix + + def __call__(self, key: str, fn: Callable): + if self.disable: + return fn() + + key = self.prefix + key + if key not in self.cache: + result = fn() + self.cache[key] = result + return self.cache[key] + + def namespace(self, namespace: str): + return Cache( + disable=self.disable, + prefix=self.prefix + namespace + ".", + cache=self.cache, + ) + +def repeat_concat( + vid: torch.FloatTensor, # (VL ... c) + txt: torch.FloatTensor, # (TL ... c) + vid_len: torch.LongTensor, # (n*b) + txt_len: torch.LongTensor, # (b) + txt_repeat: List, # (n) +) -> torch.FloatTensor: # (L ... c) + vid = torch.split(vid, vid_len.tolist()) + txt = torch.split(txt, txt_len.tolist()) + txt = [[x] * n for x, n in zip(txt, txt_repeat)] + txt = list(chain(*txt)) + return torch.cat(list(chain(*zip(vid, txt)))) + +def repeat_concat_idx( + vid_len: torch.LongTensor, # (n*b) + txt_len: torch.LongTensor, # (b) + txt_repeat: torch.LongTensor, # (n) +) -> Tuple[ + Callable, + Callable, +]: + device = vid_len.device + vid_idx = torch.arange(vid_len.sum(), device=device) + txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device) + txt_repeat_list = txt_repeat.tolist() + tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat_list) + src_idx = torch.argsort(tgt_idx) + txt_idx_len = len(tgt_idx) - len(vid_idx) + repeat_txt_len = (txt_len * txt_repeat).tolist() + + def unconcat_coalesce(all): + vid_out, txt_out = all[src_idx].split([len(vid_idx), txt_idx_len]) + txt_out_coalesced = [] + for txt, repeat_time in zip(txt_out.split(repeat_txt_len), txt_repeat_list): + txt = txt.reshape(-1, repeat_time, *txt.shape[1:]).mean(1) + txt_out_coalesced.append(txt) + return vid_out, torch.cat(txt_out_coalesced) + + return ( + lambda vid, txt: torch.cat([vid, txt])[tgt_idx], + lambda all: unconcat_coalesce(all), + ) + +def cumulative_lengths(lengths): + return [0, *accumulate(lengths)] + + +@dataclass +class MMArg: + vid: Any + txt: Any + +def get_args(key: str, args: List[Any]) -> List[Any]: + return [getattr(v, key) if isinstance(v, MMArg) else v for v in args] + + +def get_kwargs(key: str, kwargs: Dict[str, Any]) -> Dict[str, Any]: + return {k: getattr(v, key) if isinstance(v, MMArg) else v for k, v in kwargs.items()} + + +def get_window_op(name: str): + if name == "720pwin_by_size_bysize": + return make_720Pwindows_bysize + if name == "720pswin_by_size_bysize": + return make_shifted_720Pwindows_bysize + raise ValueError(f"Unknown windowing method: {name}") + + +def make_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]): + t, h, w = size + resized_nt, resized_nh, resized_nw = num_windows + scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w)) + resized_h, resized_w = round(h * scale), round(w * scale) + wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) + wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt) + nt, nh, nw = ceil(t / wt), ceil(h / wh), ceil(w / ww) + return [ + ( + slice(it * wt, min((it + 1) * wt, t)), + slice(ih * wh, min((ih + 1) * wh, h)), + slice(iw * ww, min((iw + 1) * ww, w)), + ) + for iw in range(nw) + if min((iw + 1) * ww, w) > iw * ww + for ih in range(nh) + if min((ih + 1) * wh, h) > ih * wh + for it in range(nt) + if min((it + 1) * wt, t) > it * wt + ] + +def make_shifted_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]): + t, h, w = size + resized_nt, resized_nh, resized_nw = num_windows + scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w)) + resized_h, resized_w = round(h * scale), round(w * scale) + wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) + wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt) + + st, sh, sw = ( + 0.5 if wt < t else 0, + 0.5 if wh < h else 0, + 0.5 if ww < w else 0, + ) + nt, nh, nw = ceil((t - st) / wt), ceil((h - sh) / wh), ceil((w - sw) / ww) + nt, nh, nw = ( + nt + 1 if st > 0 else 1, + nh + 1 if sh > 0 else 1, + nw + 1 if sw > 0 else 1, + ) + return [ + ( + slice(max(int((it - st) * wt), 0), min(int((it - st + 1) * wt), t)), + slice(max(int((ih - sh) * wh), 0), min(int((ih - sh + 1) * wh), h)), + slice(max(int((iw - sw) * ww), 0), min(int((iw - sw + 1) * ww), w)), + ) + for iw in range(nw) + if min(int((iw - sw + 1) * ww), w) > max(int((iw - sw) * ww), 0) + for ih in range(nh) + if min(int((ih - sh + 1) * wh), h) > max(int((ih - sh) * wh), 0) + for it in range(nt) + if min(int((it - st + 1) * wt), t) > max(int((it - st) * wt), 0) + ] + +class RotaryEmbedding(nn.Module): + def __init__( + self, + dim, + freqs_for = 'lang', + theta = 10000, + max_freq = 10, + ): + super().__init__() + + self.freqs_for = freqs_for + + if freqs_for == 'lang': + freqs = 1. / (theta ** (torch.arange(0, dim, 2)[:(dim // 2)].float() / dim)) + elif freqs_for == 'pixel': + freqs = torch.linspace(1., max_freq / 2, dim // 2) * pi + else: + raise ValueError(f"Unknown rotary frequency type: {freqs_for}") + + self.register_buffer("freqs", freqs) + + @property + def device(self): + return self.freqs.device + + def get_axial_freqs( + self, + *dims, + offsets = None + ): + Colon = slice(None) + all_freqs = [] + + if exists(offsets): + if len(offsets) != len(dims): + raise ValueError(f"SeedVR2 rotary offsets length must match dims length, got {len(offsets)} and {len(dims)}.") + + for ind, dim in enumerate(dims): + + offset = 0 + if exists(offsets): + offset = offsets[ind] + + if self.freqs_for == 'pixel': + pos = torch.linspace(-1, 1, steps = dim, device = self.device) + else: + pos = torch.arange(dim, device = self.device) + + pos = pos + offset + + freqs = self.forward(pos) + + all_axis = [None] * len(dims) + all_axis[ind] = Colon + + new_axis_slice = (Ellipsis, *all_axis, Colon) + all_freqs.append(freqs[new_axis_slice]) + + all_freqs = torch.broadcast_tensors(*all_freqs) + return torch.cat(all_freqs, dim = -1) + + def forward( + self, + t, + ): + freqs = self.freqs + + freqs = torch.einsum('..., f -> ... f', t.type(freqs.dtype), freqs) + freqs = freqs.unsqueeze(-1).expand(*freqs.shape, 2).flatten(-2) + + return freqs + +class RotaryEmbeddingBase(nn.Module): + def __init__(self, dim: int, rope_dim: int): + super().__init__() + self.rope = RotaryEmbedding( + dim=dim // rope_dim, + freqs_for="pixel", + max_freq=BYTEDANCE_ROPE_MAX_FREQ, + ) + + def get_axial_freqs(self, *dims): + return self.rope.get_axial_freqs(*dims) + + +class RotaryEmbedding3d(RotaryEmbeddingBase): + def __init__(self, dim: int): + super().__init__(dim, rope_dim=3) + self.mm = False + + +class NaRotaryEmbedding3d(RotaryEmbedding3d): + def forward( + self, + q: torch.FloatTensor, + k: torch.FloatTensor, + shape: torch.LongTensor, + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape)) + freqs = freqs.to(device=q.device) + q = q.transpose(0, 1) + k = k.transpose(0, 1) + q = _apply_seedvr2_rotary_emb(freqs, q.float()).to(q.dtype) + k = _apply_seedvr2_rotary_emb(freqs, k.float()).to(k.dtype) + q = q.transpose(0, 1) + k = k.transpose(0, 1) + return q, k + + @torch._dynamo.disable + def get_freqs( + self, + shape: torch.LongTensor, + ) -> torch.Tensor: + # Primary provenance: ByteDance-Seed/SeedVR models/dit/rope.py builds + # 7B pixel RoPE with the interleaved-angle convention, not Comfy's + # Flux freqs_cis matrix. + plain_rope = RotaryEmbedding( + dim=self.rope.freqs.numel() * 2, + freqs_for="pixel", + max_freq=BYTEDANCE_ROPE_MAX_FREQ, + ) + plain_rope = plain_rope.to(self.rope.device) + freq_list = [] + for f, h, w in shape.tolist(): + freqs = plain_rope.get_axial_freqs(f, h, w) + freq_list.append(freqs.view(-1, freqs.size(-1))) + return torch.cat(freq_list, dim=0) + + +class MMRotaryEmbeddingBase(RotaryEmbeddingBase): + def __init__(self, dim: int, rope_dim: int): + super().__init__(dim, rope_dim) + self.rope = RotaryEmbedding( + dim=dim // rope_dim, + freqs_for="lang", + theta=ROPE_THETA, + ) + self.mm = True + +def slice_at_dim(t, dim_slice: slice, *, dim): + dim += (t.ndim if dim < 0 else 0) + colons = [slice(None)] * t.ndim + colons[dim] = dim_slice + return t[tuple(colons)] + +def rotate_half(x): + x = x.reshape(*x.shape[:-1], x.shape[-1] // 2, 2) + x1, x2 = x.unbind(dim = -1) + x = torch.stack((-x2, x1), dim = -1) + return x.flatten(-2) +def exists(val): + return val is not None + +def _apply_seedvr2_rotary_emb( + freqs: torch.Tensor, + t: torch.Tensor, + start_index: int = 0, + scale: float = 1.0, + seq_dim: int = -2, + freqs_seq_dim: int | None = None, +) -> torch.Tensor: + dtype = t.dtype + if freqs_seq_dim is None and (freqs.ndim == 2 or t.ndim == 3): + freqs_seq_dim = 0 + + if t.ndim == 3 or freqs_seq_dim is not None: + seq_len = t.shape[seq_dim] + freqs = slice_at_dim(freqs, slice(-seq_len, None), dim=freqs_seq_dim) + + rot_feats = freqs.shape[-1] + end_index = start_index + rot_feats + + t_left = t[..., :start_index] + t_middle = t[..., start_index:end_index] + t_right = t[..., end_index:] + + freqs = freqs.to(device=t_middle.device, dtype=t_middle.dtype) + cos = freqs.cos() * scale + sin = freqs.sin() * scale + t_middle = (t_middle * cos) + (rotate_half(t_middle) * sin) + return torch.cat((t_left, t_middle, t_right), dim=-1).to(dtype) + +def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor: + angles = freqs_interleaved[..., ::2].float() + cos = torch.cos(angles) + sin = torch.sin(angles) + out = torch.stack([cos, -sin, sin, cos], dim=-1) + return out.reshape(*out.shape[:-1], 2, 2) + + +def _apply_rope1_partial(t: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + out = t.clone() if t.requires_grad or comfy.model_management.in_training else t + rot_d = 2 * freqs_cis.shape[-3] + seq_len = out.shape[-2] + for start in range(0, seq_len, SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS): + end = min(start + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, seq_len) + freqs_chunk = freqs_cis[start:end] + if rot_d == out.shape[-1]: + out[..., start:end, :] = apply_rope1(out[..., start:end, :], freqs_chunk).to(out.dtype) + else: + out[..., start:end, :rot_d] = apply_rope1(out[..., start:end, :rot_d], freqs_chunk).to(out.dtype) + return out + + +class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase): + def __init__(self, dim: int): + super().__init__(dim, rope_dim=3) + + def forward( + self, + vid_q: torch.FloatTensor, # L h d + vid_k: torch.FloatTensor, # L h d + vid_shape: torch.LongTensor, # B 3 + txt_q: torch.FloatTensor, # L h d + txt_k: torch.FloatTensor, # L h d + txt_shape: torch.LongTensor, # B 1 + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_freqs, txt_freqs = cache( + "mmrope_freqs_3d", + lambda: self.get_freqs(vid_shape, txt_shape), + ) + target_device = vid_q.device + if vid_freqs.device != target_device: + vid_freqs = vid_freqs.to(target_device) + if txt_freqs.device != target_device: + txt_freqs = txt_freqs.to(target_device) + vid_q = vid_q.transpose(0, 1) + vid_k = vid_k.transpose(0, 1) + vid_q = _apply_rope1_partial(vid_q, vid_freqs) + vid_k = _apply_rope1_partial(vid_k, vid_freqs) + vid_q = vid_q.transpose(0, 1) + vid_k = vid_k.transpose(0, 1) + + txt_q = txt_q.transpose(0, 1) + txt_k = txt_k.transpose(0, 1) + txt_q = _apply_rope1_partial(txt_q, txt_freqs) + txt_k = _apply_rope1_partial(txt_k, txt_freqs) + txt_q = txt_q.transpose(0, 1) + txt_k = txt_k.transpose(0, 1) + return vid_q, vid_k, txt_q, txt_k + + @torch._dynamo.disable # Disable compilation: .tolist() is data-dependent and causes graph breaks + def get_freqs( + self, + vid_shape: torch.LongTensor, + txt_shape: torch.LongTensor, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + ]: + + max_temporal = 0 + max_height = 0 + max_width = 0 + max_txt_len = 0 + + for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()): + max_temporal = max(max_temporal, l + f) + max_height = max(max_height, h) + max_width = max(max_width, w) + max_txt_len = max(max_txt_len, l) + + autocast_device = "cuda" if torch.cuda.is_available() else "cpu" + with torch.amp.autocast(autocast_device, enabled=False): + vid_freqs = self.get_axial_freqs( + max_temporal + 16, + max_height + 4, + max_width + 4, + ).float() + txt_freqs = self.get_axial_freqs(max_txt_len + 16) + + vid_freq_list, txt_freq_list = [], [] + for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()): + vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1)) + txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1)) + vid_freq_list.append(vid_freq) + txt_freq_list.append(txt_freq) + vid_freqs_interleaved = torch.cat(vid_freq_list, dim=0) + txt_freqs_interleaved = torch.cat(txt_freq_list, dim=0) + + return _to_flux_freqs_cis(vid_freqs_interleaved), _to_flux_freqs_cis(txt_freqs_interleaved) + +class MMModule(nn.Module): + def __init__( + self, + module: Callable[..., nn.Module], + *args, + shared_weights: bool = False, + vid_only: bool = False, + **kwargs, + ): + super().__init__() + self.shared_weights = shared_weights + self.vid_only = vid_only + if self.shared_weights: + if get_args("vid", args) != get_args("txt", args): + raise ValueError("SeedVR2 shared MMModule requires matching vid/txt args.") + if get_kwargs("vid", kwargs) != get_kwargs("txt", kwargs): + raise ValueError("SeedVR2 shared MMModule requires matching vid/txt kwargs.") + self.all = module(*get_args("vid", args), **get_kwargs("vid", kwargs)) + else: + self.vid = module(*get_args("vid", args), **get_kwargs("vid", kwargs)) + self.txt = ( + module(*get_args("txt", args), **get_kwargs("txt", kwargs)) + if not vid_only + else None + ) + + def forward( + self, + vid: torch.FloatTensor, + txt: torch.FloatTensor, + *args, + **kwargs, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_module = self.vid if not self.shared_weights else self.all + vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs)) + if not self.vid_only: + txt_module = self.txt if not self.shared_weights else self.all + txt = txt.to(device=vid.device, dtype=vid.dtype) + txt = txt_module(txt, *get_args("txt", args), **get_kwargs("txt", kwargs)) + return vid, txt + +def get_na_rope(rope_type: Optional[str], dim: int): + if rope_type is None: + return None + if rope_type == "rope3d": + return NaRotaryEmbedding3d(dim=dim) + if rope_type == "mmrope3d": + return NaMMRotaryEmbedding3d(dim=dim) + raise ValueError(f"Unknown SeedVR2 rope type: {rope_type}") + +class NaMMAttention(nn.Module): + def __init__( + self, + vid_dim: int, + txt_dim: int, + heads: int, + head_dim: int, + qk_bias: bool, + qk_norm, + qk_norm_eps: float, + rope_type: Optional[str], + rope_dim: int, + shared_weights: bool, + device, dtype, operations, + ): + super().__init__() + dim = MMArg(vid_dim, txt_dim) + self.heads = heads + inner_dim = heads * head_dim + qkv_dim = inner_dim * 3 + self.head_dim = head_dim + self.proj_qkv = MMModule( + operations.Linear, dim, qkv_dim, bias=qk_bias, shared_weights=shared_weights, device=device, dtype=dtype + ) + self.proj_out = MMModule(operations.Linear, inner_dim, dim, shared_weights=shared_weights, device=device, dtype=dtype) + self.norm_q = MMModule( + qk_norm, + normalized_shape=head_dim, + eps=qk_norm_eps, + elementwise_affine=True, + shared_weights=shared_weights, + device=device, dtype=dtype + ) + self.norm_k = MMModule( + qk_norm, + normalized_shape=head_dim, + eps=qk_norm_eps, + elementwise_affine=True, + shared_weights=shared_weights, + device=device, dtype=dtype + ) + + + self.rope = get_na_rope(rope_type=rope_type, dim=rope_dim) + +def window( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + window_fn: Callable[[torch.Tensor], List[torch.Tensor]], +): + hid = unflatten(hid, hid_shape) + hid = list(map(window_fn, hid)) + hid_windows_list = [len(x) for x in hid] + hid_windows = torch.as_tensor(hid_windows_list, device=hid_shape.device) + hid = list(chain(*hid)) + hid_len_list = [math.prod(x.shape[:-1]) for x in hid] + hid, hid_shape = flatten(hid) + return hid, hid_shape, hid_windows, hid_len_list, hid_windows_list + +def window_idx( + hid_shape: torch.LongTensor, # (b n) + window_fn: Callable[[torch.Tensor], List[torch.Tensor]], +): + hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1) + tgt_idx, tgt_shape, tgt_windows, tgt_len_list, tgt_windows_list = window(hid_idx, hid_shape, window_fn) + tgt_idx = tgt_idx.squeeze(-1) + src_idx = torch.argsort(tgt_idx) + return ( + lambda hid: torch.index_select(hid, 0, tgt_idx), + lambda hid: torch.index_select(hid, 0, src_idx), + tgt_shape, + tgt_windows, + tgt_len_list, + tgt_windows_list, + ) + +class NaSwinAttention(NaMMAttention): + def __init__( + self, + *args, + window: Union[int, Tuple[int, int, int]], + window_method: str, + version: bool = False, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.version_7b = version + self.window = _triple(window) + self.window_method = window_method + if not all(isinstance(v, int) and v >= 0 for v in self.window): + raise ValueError(f"SeedVR2 window must contain non-negative integers, got {self.window}.") + + self.window_op = get_window_op(window_method) + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + + vid_qkv, txt_qkv = self.proj_qkv(vid, txt) + + cache_win = cache.namespace(f"{self.window_method}_{self.window}_sd3") + + def make_window(x: torch.Tensor): + t, h, w, _ = x.shape + window_slices = self.window_op((t, h, w), self.window) + return [x[st, sh, sw] for (st, sh, sw) in window_slices] + + window_partition, window_reverse, window_shape, window_count, vid_len_win_list, window_count_list = cache_win( + "win_transform", + lambda: window_idx(vid_shape, make_window), + ) + vid_qkv_win = window_partition(vid_qkv) + + vid_qkv_win = vid_qkv_win.reshape(vid_qkv_win.shape[0], 3, self.heads, self.head_dim) + txt_qkv = txt_qkv.reshape(txt_qkv.shape[0], 3, self.heads, self.head_dim) + + vid_q, vid_k, vid_v = vid_qkv_win.unbind(1) + txt_q, txt_k, txt_v = txt_qkv.unbind(1) + + vid_q, txt_q = self.norm_q(vid_q, txt_q) + vid_k, txt_k = self.norm_k(vid_k, txt_k) + + txt_len = cache("txt_len", lambda: txt_shape.prod(-1)) + + vid_len_win = cache_win("vid_len", lambda: window_shape.prod(-1)) + txt_len = txt_len.to(window_count.device) + + if self.rope: + if self.version_7b: + vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win) + elif self.rope.mm: + _, num_h, _ = txt_q.shape + txt_q_repeat = txt_q.flatten(1, 2) + txt_q_repeat = unflatten(txt_q_repeat, txt_shape) + txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count_list)] + txt_q_repeat = list(chain(*txt_q_repeat)) + txt_q_repeat, txt_shape_repeat = flatten(txt_q_repeat) + txt_q_repeat = txt_q_repeat.reshape(txt_q_repeat.shape[0], num_h, self.head_dim) + + txt_k_repeat = txt_k.flatten(1, 2) + txt_k_repeat = unflatten(txt_k_repeat, txt_shape) + txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count_list)] + txt_k_repeat = list(chain(*txt_k_repeat)) + txt_k_repeat, _ = flatten(txt_k_repeat) + txt_k_repeat = txt_k_repeat.reshape(txt_k_repeat.shape[0], num_h, self.head_dim) + + vid_q, vid_k, txt_q, txt_k = self.rope( + vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win + ) + else: + vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win) + + txt_len_win_list = cache_win( + "txt_len_list", + lambda: [txt_len for txt_len, window_count in zip(txt_len.tolist(), window_count_list) for _ in range(window_count)], + ) + all_len_win = cache_win("all_len", lambda: [vid_len + txt_len for vid_len, txt_len in zip(vid_len_win_list, txt_len_win_list)]) + concat_win, unconcat_win = cache_win( + "mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count) + ) + out = optimized_var_attention( + q=concat_win(vid_q, txt_q), + k=concat_win(vid_k, txt_k), + v=concat_win(vid_v, txt_v), + heads=self.heads, skip_reshape=True, skip_output_reshape=True, + cu_seqlens_q=cache_win("vid_seqlens_q", lambda: cumulative_lengths(all_len_win)), + cu_seqlens_k=cache_win("vid_seqlens_k", lambda: cumulative_lengths(all_len_win)), + ) + vid_out, txt_out = unconcat_win(out) + + vid_out = vid_out.flatten(1, 2) + txt_out = txt_out.flatten(1, 2) + vid_out = window_reverse(vid_out) + + vid_out, txt_out = self.proj_out(vid_out, txt_out) + + return vid_out, txt_out + +class MLP(nn.Module): + def __init__( + self, + dim: int, + expand_ratio: int, + device, dtype, operations + ): + super().__init__() + self.proj_in = operations.Linear(dim, dim * expand_ratio, device=device, dtype=dtype) + self.act = nn.GELU("tanh") + self.proj_out = operations.Linear(dim * expand_ratio, dim, device=device, dtype=dtype) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + x = self.proj_in(x) + x = self.act(x) + x = self.proj_out(x) + return x + + +class SwiGLUMLP(nn.Module): + def __init__( + self, + dim: int, + expand_ratio: int, + multiple_of: int = 256, + device=None, dtype=None, operations=None + ): + super().__init__() + hidden_dim = int(2 * dim * expand_ratio / 3) + hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) + self.proj_in_gate = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype) + self.proj_out = operations.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype) + self.proj_in = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + return self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x)) + +def get_mlp(mlp_type: Optional[str] = "normal"): + if mlp_type == "normal": + return MLP + if mlp_type == "swiglu": + return SwiGLUMLP + raise ValueError(f"Unknown SeedVR2 MLP type: {mlp_type}") + +class NaMMSRTransformerBlock(nn.Module): + def __init__( + self, + *, + vid_dim: int, + txt_dim: int, + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm, + norm_eps: float, + ada, + qk_bias: bool, + qk_norm, + mlp_type: str, + shared_weights: bool, + rope_type: str, + rope_dim: int, + is_last_layer: bool, + window: Union[int, Tuple[int, int, int]], + window_method: str, + version: bool, + device, dtype, operations, + ): + super().__init__() + dim = MMArg(vid_dim, txt_dim) + self.attn_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, device=device, dtype=dtype) + + self.attn = NaSwinAttention( + vid_dim=vid_dim, + txt_dim=txt_dim, + heads=heads, + head_dim=head_dim, + qk_bias=qk_bias, + qk_norm=qk_norm, + qk_norm_eps=norm_eps, + rope_type=rope_type, + rope_dim=rope_dim, + shared_weights=shared_weights, + window=window, + window_method=window_method, + version=version, + device=device, dtype=dtype, operations=operations + ) + + self.mlp_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype) + self.mlp = MMModule( + get_mlp(mlp_type), + dim=dim, + expand_ratio=expand_ratio, + shared_weights=shared_weights, + vid_only=is_last_layer, + device=device, dtype=dtype, operations=operations + ) + self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype) + self.is_last_layer = is_last_layer + self.version = version + + def _seedvr2_7b_mlp( + self, + vid: torch.FloatTensor, + txt: torch.FloatTensor, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_module = self.mlp.vid if not self.mlp.shared_weights else self.mlp.all + if comfy.model_management.in_training or vid.requires_grad: + vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0)], dim=0) + else: + vid_out = None + offset = 0 + for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0): + chunk_out = vid_module(chunk) + if vid_out is None: + vid_out = chunk_out.new_empty((vid.shape[0], *chunk_out.shape[1:])) + vid_out[offset:offset + chunk_out.shape[0]] = chunk_out + offset += chunk_out.shape[0] + vid = vid_out + if not self.mlp.vid_only: + txt_module = self.mlp.txt if not self.mlp.shared_weights else self.mlp.all + txt = txt.to(device=vid.device, dtype=vid.dtype) + txt = txt_module(txt) + return vid, txt + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + emb: torch.FloatTensor, + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + torch.LongTensor, + torch.LongTensor, + ]: + hid_len = MMArg( + cache("vid_len", lambda: vid_shape.prod(-1)), + cache("txt_len", lambda: txt_shape.prod(-1)), + ) + ada_kwargs = { + "emb": emb, + "hid_len": hid_len, + "cache": cache, + "branch_tag": MMArg("vid", "txt"), + } + + vid_attn, txt_attn = self.attn_norm(vid, txt) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs) + vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs) + vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt) + + vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs) + if self.version: + vid_mlp, txt_mlp = self._seedvr2_7b_mlp(vid_mlp, txt_mlp) + else: + vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs) + vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn) + + return vid_mlp, txt_mlp, vid_shape, txt_shape + +class PatchOut(nn.Module): + def __init__( + self, + out_channels: int, + patch_size: Union[int, Tuple[int, int, int]], + dim: int, + device, dtype, operations + ): + super().__init__() + t, h, w = _triple(patch_size) + self.patch_size = t, h, w + self.proj = operations.Linear(dim, out_channels * t * h * w, device=device, dtype=dtype) + + def forward( + self, + vid: torch.Tensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + vid = self.proj(vid) + b, T, H, W, channels = vid.shape + c = channels // (t * h * w) + vid = vid.view(b, T, H, W, t, h, w, c).permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(b, c, T * t, H * h, W * w) + if t > 1: + vid = vid[:, :, (t - 1) :] + return vid + +class NaPatchOut(PatchOut): + def forward( + self, + vid: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, + cache: Optional[Cache] = None, + vid_shape_before_patchify = None + ) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, + ]: + if cache is None: + cache = Cache(disable=True) + + t, h, w = self.patch_size + vid = self.proj(vid) + + if not (t == h == w == 1): + vid = unflatten(vid, vid_shape) + for i in range(len(vid)): + T, H, W, channels = vid[i].shape + c = channels // (t * h * w) + vid[i] = vid[i].view(T, H, W, t, h, w, c).permute(0, 3, 1, 4, 2, 5, 6).reshape(T * t, H * h, W * w, c) + if t > 1 and vid_shape_before_patchify[i, 0] % t != 0: + vid[i] = vid[i][(t - vid_shape_before_patchify[i, 0] % t) :] + vid, vid_shape = flatten(vid) + + return vid, vid_shape + +class PatchIn(nn.Module): + def __init__( + self, + in_channels: int, + patch_size: Union[int, Tuple[int, int, int]], + dim: int, + device, dtype, operations + ): + super().__init__() + t, h, w = _triple(patch_size) + self.patch_size = t, h, w + self.proj = operations.Linear(in_channels * t * h * w, dim, device=device, dtype=dtype) + + def forward( + self, + vid: torch.Tensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + if t > 1: + if vid.size(2) % t != 1: + raise ValueError( + f"SeedVR2 patch input temporal size must satisfy T % {t} == 1, got {vid.size(2)}." + ) + vid = torch.cat([vid[:, :, :1]] * (t - 1) + [vid], dim=2) + b, c, Tt, Hh, Ww = vid.shape + vid = vid.view(b, c, Tt // t, t, Hh // h, h, Ww // w, w).permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(b, Tt // t, Hh // h, Ww // w, t * h * w * c) + vid = self.proj(vid) + return vid + +class NaPatchIn(PatchIn): + def forward( + self, + vid: torch.Tensor, # l c + vid_shape: torch.LongTensor, + cache: Optional[Cache] = None, + ) -> torch.Tensor: + if cache is None: + cache = Cache(disable=True) + cache = cache.namespace("patch") + vid_shape_before_patchify = cache("vid_shape_before_patchify", lambda: vid_shape) + t, h, w = self.patch_size + if not (t == h == w == 1): + vid = unflatten(vid, vid_shape) + for i in range(len(vid)): + if t > 1 and vid_shape_before_patchify[i, 0] % t != 0: + vid[i] = torch.cat([vid[i][:1]] * (t - vid[i].size(0) % t) + [vid[i]], dim=0) + Tt, Hh, Ww, c = vid[i].shape + vid[i] = vid[i].view(Tt // t, t, Hh // h, h, Ww // w, w, c).permute(0, 2, 4, 1, 3, 5, 6).reshape(Tt // t, Hh // h, Ww // w, t * h * w * c) + vid, vid_shape = flatten(vid) + + vid = self.proj(vid) + return vid, vid_shape + +def expand_dims(x: torch.Tensor, dim: int, ndim: int): + shape = x.shape + shape = shape[:dim] + (1,) * (ndim - len(shape)) + shape[dim:] + return x.reshape(shape) + + +class AdaSingle(nn.Module): + def __init__( + self, + dim: int, + emb_dim: int, + layers: List[str], + modes: Tuple[str, ...] = ("in", "out"), + device = None, dtype = None, + ): + if emb_dim != 6 * dim: + raise ValueError(f"SeedVR2 AdaSingle requires emb_dim == 6 * dim, got emb_dim={emb_dim}, dim={dim}.") + super().__init__() + self.dim = dim + self.emb_dim = emb_dim + self.layers = layers + + param_kwargs = {"device": device, "dtype": dtype} + + for l in layers: + if "in" in modes: + self.register_parameter(f"{l}_shift", nn.Parameter(torch.empty(dim, **param_kwargs))) + self.register_parameter(f"{l}_scale", nn.Parameter(torch.empty(dim, **param_kwargs))) + if "out" in modes: + self.register_parameter(f"{l}_gate", nn.Parameter(torch.empty(dim, **param_kwargs))) + + def forward( + self, + hid: torch.FloatTensor, # b ... c + emb: torch.FloatTensor, # b d + layer: str, + mode: str, + cache: Optional[Cache] = None, + branch_tag: str = "", + hid_len: Optional[torch.LongTensor] = None, # b + ) -> torch.FloatTensor: + if cache is None: + cache = Cache(disable=True) + idx = self.layers.index(layer) + emb = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :] + emb = expand_dims(emb, 1, hid.ndim + 1) + + if hid_len is not None: + emb = cache( + f"emb_repeat_{idx}_{branch_tag}", + lambda: torch.repeat_interleave(emb, hid_len, dim=0), + ) + + shiftA, scaleA, gateA = emb.unbind(-1) + shiftB, scaleB, gateB = ( + getattr(self, f"{layer}_shift", None), + getattr(self, f"{layer}_scale", None), + getattr(self, f"{layer}_gate", None), + ) + + if mode == "in": + shiftB = comfy.ops.cast_to_input(shiftB, hid) + scaleB = comfy.ops.cast_to_input(scaleB, hid) + return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB) + if mode == "out": + if gateB is not None: + gateB = comfy.ops.cast_to_input(gateB, hid) + return hid.mul_(gateA + gateB) + else: + return hid.mul_(gateA) + + raise ValueError(f"Unknown AdaSingle mode: {mode}") + + +class TimeEmbedding(nn.Module): + def __init__( + self, + sinusoidal_dim: int, + hidden_dim: int, + output_dim: int, + device, dtype, operations + ): + super().__init__() + self.sinusoidal_dim = sinusoidal_dim + self.proj_in = operations.Linear(sinusoidal_dim, hidden_dim, device=device, dtype=dtype) + self.proj_hid = operations.Linear(hidden_dim, hidden_dim, device=device, dtype=dtype) + self.proj_out = operations.Linear(hidden_dim, output_dim, device=device, dtype=dtype) + self.act = nn.SiLU() + + def forward( + self, + timestep: Union[int, float, torch.IntTensor, torch.FloatTensor], + device: torch.device, + dtype: torch.dtype, + ) -> torch.FloatTensor: + if not torch.is_tensor(timestep): + timestep = torch.tensor([timestep], device=device, dtype=dtype) + if timestep.ndim == 0: + timestep = timestep[None] + + emb = get_timestep_embedding( + timesteps=timestep, + embedding_dim=self.sinusoidal_dim, + flip_sin_to_cos=False, + downscale_freq_shift=0, + ).to(dtype) + emb = self.proj_in(emb) + emb = self.act(emb) + emb = self.proj_hid(emb) + emb = self.act(emb) + emb = self.proj_out(emb) + return emb + +def flatten( + hid: List[torch.FloatTensor], # List of (*** c) +) -> Tuple[ + torch.FloatTensor, # (L c) + torch.LongTensor, # (b n) +]: + if len(hid) == 0: + raise ValueError("SeedVR2 flatten requires at least one tensor.") + shape = torch.as_tensor([x.shape[:-1] for x in hid], device=hid[0].device) + hid = torch.cat([x.flatten(0, -2) for x in hid]) + return hid, shape + + +def unflatten( + hid: torch.FloatTensor, # (L c) or (L ... c) + hid_shape: torch.LongTensor, # (b n) +) -> List[torch.Tensor]: # List of (*** c) or (*** ... c) + hid_len = hid_shape.prod(-1) + hid = hid.split(hid_len.tolist()) + hid = [x.unflatten(0, s.tolist()) for x, s in zip(hid, hid_shape)] + return hid + +class NaDiT(nn.Module): + + def __init__( + self, + norm_eps, + num_layers, + mlp_type, + vid_in_channels = 33, + vid_out_channels = SEEDVR2_LATENT_CHANNELS, + vid_dim = 2560, + txt_in_dim = 5120, + heads = 20, + head_dim = 128, + mm_layers = 10, + expand_ratio = 4, + qk_bias = False, + patch_size = (1, 2, 2), + rope_dim = 128, + rope_type = "mmrope3d", + vid_out_norm: Optional[str] = None, + image_model = None, + device = None, + dtype = None, + operations = None, + ): + if image_model not in (None, "seedvr2"): + raise ValueError(f"SeedVR2 NaDiT expected image_model='seedvr2', got {image_model!r}.") + self._7b_version = vid_dim == SEEDVR2_7B_VID_DIM + if self._7b_version: + rope_type = "rope3d" + self.dtype = dtype + factory_kwargs = {"device": device, "dtype": dtype} + window_method = num_layers // 2 * ["720pwin_by_size_bysize","720pswin_by_size_bysize"] + txt_dim = vid_dim + emb_dim = vid_dim * 6 + window = num_layers * [(4,3,3)] + ada = AdaSingle + norm = operations.RMSNorm + qk_norm = operations.RMSNorm + super().__init__() + self.register_buffer("positive_conditioning", torch.empty((58, 5120), device=device, dtype=dtype)) + self.register_buffer("negative_conditioning", torch.empty((64, 5120), device=device, dtype=dtype)) + self.vid_in = NaPatchIn( + in_channels=vid_in_channels, + patch_size=patch_size, + dim=vid_dim, + device=device, dtype=dtype, operations=operations + ) + self.txt_in = ( + operations.Linear(txt_in_dim, txt_dim, **factory_kwargs) + if txt_in_dim and txt_in_dim != txt_dim + else nn.Identity() + ) + self.emb_in = TimeEmbedding( + sinusoidal_dim=BYTEDANCE_SINUSOIDAL_DIM, + hidden_dim=max(vid_dim, txt_dim), + output_dim=emb_dim, + device=device, dtype=dtype, operations=operations + ) + + if window is None or isinstance(window[0], int): + window = [window] * num_layers + + rope_dim = rope_dim if rope_dim is not None else head_dim // 2 + self.blocks = nn.ModuleList( + [ + NaMMSRTransformerBlock( + vid_dim=vid_dim, + txt_dim=txt_dim, + emb_dim=emb_dim, + heads=heads, + head_dim=head_dim, + expand_ratio=expand_ratio, + norm=norm, + norm_eps=norm_eps, + ada=ada, + qk_bias=qk_bias, + qk_norm=qk_norm, + mlp_type=mlp_type, + rope_dim = rope_dim, + window=window[i], + window_method=window_method[i], + version = self._7b_version, + is_last_layer=(i == num_layers - 1) and not self._7b_version, + rope_type = rope_type, + shared_weights=not ( + (i < mm_layers) if isinstance(mm_layers, int) else mm_layers[i] + ), + operations = operations, + **factory_kwargs + ) + for i in range(num_layers) + ] + ) + self.vid_out = NaPatchOut( + out_channels=vid_out_channels, + patch_size=patch_size, + dim=vid_dim, + device=device, dtype=dtype, operations=operations + ) + + self.vid_out_norm = None + if vid_out_norm is not None: + self.vid_out_norm = operations.RMSNorm( + normalized_shape=vid_dim, + eps=norm_eps, + elementwise_affine=True, + device=device, dtype=dtype + ) + self.vid_out_ada = ada( + dim=vid_dim, + emb_dim=emb_dim, + layers=["out"], + modes=["in"], + device=device, dtype=dtype + ) + + def _resolve_text_conditioning(self, context, cond_or_uncond=None): + if context is None or context.numel() == 0: + context = self.positive_conditioning + return flatten([context]) + if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond): + if context.shape[0] == 1: + context = context.squeeze(0) + return flatten([context]) + return flatten(context.unbind(0)) + if context.shape[0] % 2 != 0: + raise ValueError(f"SeedVR2 expected an even text-conditioning batch, got shape {tuple(context.shape)}") + neg_cond, pos_cond = context.chunk(2, dim=0) + if pos_cond.shape[0] == 1: + pos_cond, neg_cond = pos_cond.squeeze(0), neg_cond.squeeze(0) + return flatten([pos_cond, neg_cond]) + return flatten((*pos_cond.unbind(0), *neg_cond.unbind(0))) + + @staticmethod + def _seedvr2_is_single_conditioning_branch(cond_or_uncond): + if cond_or_uncond is None or len(cond_or_uncond) == 0: + return False + first = cond_or_uncond[0] + return all(entry == first for entry in cond_or_uncond) + + @staticmethod + def _check_seedvr2_video_latent(x, channels, name): + if x.ndim != 5: + raise ValueError(f"SeedVR2 expected {name} to be 5-D native latent, got shape {tuple(x.shape)}.") + if x.shape[1] != channels: + raise ValueError(f"SeedVR2 expected {name} channels to be {channels}, got shape {tuple(x.shape)}.") + return x + + def _swap_pos_neg_halves(self, out, cond_or_uncond=None): + if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond): + return out + pos, neg = out.chunk(2, dim=0) + return torch.cat([neg, pos], dim=0) + + def forward( + self, + x, + timestep, + context, # l c + disable_cache: bool = False, + **kwargs + ): + transformer_options = kwargs.get("transformer_options", {}) + patches_replace = transformer_options.get("patches_replace", {}) + blocks_replace = patches_replace.get("dit", {}) + conditions = kwargs.get("condition") + if conditions is None: + raise ValueError("SeedVR2 requires conditioning latents from the SeedVR2Conditioning node.") + x = self._check_seedvr2_video_latent(x, SEEDVR2_LATENT_CHANNELS, "latent") + conditions = self._check_seedvr2_video_latent(conditions, SEEDVR2_LATENT_CHANNELS + 1, "conditioning") + b, _, t, h, w = x.shape + if conditions.shape[0] != b or conditions.shape[2:] != (t, h, w): + raise ValueError( + f"SeedVR2 conditioning shape must match latent batch/temporal/spatial dimensions; got latent {tuple(x.shape)} and conditioning {tuple(conditions.shape)}." + ) + x = x.movedim(1, -1) + conditions = conditions.movedim(1, -1) + cache = Cache(disable=disable_cache) + + txt, txt_shape = self._resolve_text_conditioning(context, transformer_options.get("cond_or_uncond")) + + vid, vid_shape = flatten(x) + cond_latent, _ = flatten(conditions) + + vid = torch.cat([vid, cond_latent], dim=-1) + + txt = self.txt_in(txt) + + vid_shape_before_patchify = vid_shape + vid, vid_shape = self.vid_in(vid, vid_shape, cache=cache) + + emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype) + + for i, block in enumerate(self.blocks): + if ("block", i) in blocks_replace: + def block_wrap(args): + out = {} + out["vid"], out["txt"], out["vid_shape"], out["txt_shape"] = block( + vid=args["vid"], + txt=args["txt"], + vid_shape=args["vid_shape"], + txt_shape=args["txt_shape"], + emb=args["emb"], + cache=args["cache"], + ) + return out + out = blocks_replace[("block", i)]({ + "vid":vid, + "txt":txt, + "vid_shape":vid_shape, + "txt_shape":txt_shape, + "emb":emb, + "cache":cache, + }, {"original_block": block_wrap}) + vid, txt, vid_shape, txt_shape = out["vid"], out["txt"], out["vid_shape"], out["txt_shape"] + else: + vid, txt, vid_shape, txt_shape = block( + vid=vid, + txt=txt, + vid_shape=vid_shape, + txt_shape=txt_shape, + emb=emb, + cache=cache, + ) + + if self.vid_out_norm: + vid = self.vid_out_norm(vid) + vid = self.vid_out_ada( + vid, + emb=emb, + layer="out", + mode="in", + hid_len=cache("vid_len", lambda: vid_shape.prod(-1)), + cache=cache, + branch_tag="vid", + ) + + vid, vid_shape = self.vid_out(vid, vid_shape, cache, vid_shape_before_patchify = vid_shape_before_patchify) + vid = unflatten(vid, vid_shape) + out = torch.stack(vid) + out = out.movedim(-1, 1) + return self._swap_pos_neg_halves(out, transformer_options.get("cond_or_uncond")) diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.py new file mode 100644 index 000000000..7a8070b65 --- /dev/null +++ b/comfy/ldm/seedvr/vae.py @@ -0,0 +1,1610 @@ +from typing import Literal, Optional, Tuple +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from contextlib import contextmanager +from comfy.utils import ProgressBar + +from comfy.ldm.seedvr.constants import ( + BYTEDANCE_BLOCK_OUT_CHANNELS, + BYTEDANCE_GN_CHUNKS_FP16, + BYTEDANCE_GN_CHUNKS_FP32, + BYTEDANCE_LOGVAR_CLAMP_MAX, + BYTEDANCE_LOGVAR_CLAMP_MIN, + BYTEDANCE_SLICING_SAMPLE_MIN, + BYTEDANCE_VAE_CONV_MEM_GIB, + BYTEDANCE_VAE_NORM_MEM_GIB, + BYTEDANCE_VAE_SCALING_FACTOR, + BYTEDANCE_VAE_SHIFTING_FACTOR, + BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE, + BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE, + SEEDVR2_LATENT_CHANNELS, +) +from comfy.ldm.modules.attention import optimized_attention +from comfy.ldm.modules.diffusionmodules.model import vae_attention + +import math +from enum import Enum + +import logging +import comfy.model_management +import comfy.ops +ops = comfy.ops.manual_cast + + +def _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, temporal_scale=1): + if temporal_size is None: + return None + + temporal_size = int(temporal_size) + if temporal_size <= 0: + return None + + temporal_overlap = max(0, int(temporal_overlap or 0)) + temporal_overlap = min(temporal_overlap, temporal_size - 1) + temporal_step = temporal_size - temporal_overlap + temporal_scale = max(1, int(temporal_scale)) + return max(1, math.ceil(temporal_step / temporal_scale)) + + +def _seedvr2_clamped_spatial_overlap(overlap, tile_size): + overlap = max(0, int(overlap)) + tile_size = max(1, int(tile_size)) + return min(overlap, tile_size - 1) + + +def tiled_vae( + x, + vae_model, + tile_size=(512, 512), + tile_overlap=(64, 64), + temporal_size=16, + temporal_overlap=0, + encode=True, +): + if x.ndim != 5: + x = x.unsqueeze(2) + + _, _, d, h, w = x.shape + + sf_s = getattr(vae_model, "spatial_downsample_factor", BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE) + sf_t = getattr(vae_model, "temporal_downsample_factor", BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE) + if encode: + slicing_attr = "slicing_sample_min_size" + slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap) + else: + slicing_attr = "slicing_latent_min_size" + slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, sf_t) + if encode: + ti_h, ti_w = tile_size + ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0], ti_h) + ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1], ti_w) + blend_ov_h = max(0, ov_h // sf_s) + blend_ov_w = max(0, ov_w // sf_s) + target_d = (d + sf_t - 1) // sf_t + target_h = (h + sf_s - 1) // sf_s + target_w = (w + sf_s - 1) // sf_s + else: + ti_h = max(1, tile_size[0] // sf_s) + ti_w = max(1, tile_size[1] // sf_s) + ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0] // sf_s, ti_h) + ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1] // sf_s, ti_w) + blend_ov_h = ov_h * sf_s + blend_ov_w = ov_w * sf_s + + target_d = max(1, d * sf_t - (sf_t - 1)) + target_h = h * sf_s + target_w = w * sf_s + + stride_h = max(1, ti_h - ov_h) + stride_w = max(1, ti_w - ov_w) + + storage_device = vae_model.device + result = None + count = None + def run_temporal_chunks(spatial_tile, model=vae_model): + t_chunk = spatial_tile.contiguous() + old_device = getattr(model, "device", None) + model.device = t_chunk.device + old_slicing_min_size = getattr(model, slicing_attr, None) + if old_slicing_min_size is not None and slicing_min_size is not None: + if slicing_min_size <= 0: + setattr(model, slicing_attr, t_chunk.shape[2]) + else: + setattr(model, slicing_attr, slicing_min_size) + try: + if encode: + out = model.encode(t_chunk) + else: + out = model.decode_(t_chunk) + finally: + if old_slicing_min_size is not None and slicing_min_size is not None: + setattr(model, slicing_attr, old_slicing_min_size) + if old_device is not None: + model.device = old_device + if out.ndim == 4: + out = out.unsqueeze(2) + return out.to(storage_device) + + ramp_cache = {} + def get_ramp(steps): + if steps not in ramp_cache: + t = torch.linspace(0, 1, steps=steps, device=storage_device, dtype=torch.float32) + ramp_cache[steps] = 0.5 - 0.5 * torch.cos(t * torch.pi) + return ramp_cache[steps] + + tile_ranges = [] + for y_idx in range(0, h, stride_h): + y_end = min(y_idx + ti_h, h) + if y_idx > 0 and (y_end - y_idx) <= ov_h: + continue + for x_idx in range(0, w, stride_w): + x_end = min(x_idx + ti_w, w) + if x_idx > 0 and (x_end - x_idx) <= ov_w: + continue + tile_ranges.append((y_idx, y_end, x_idx, x_end)) + + total_tiles = len(tile_ranges) + bar = ProgressBar(total_tiles) + single_spatial_tile = h <= ti_h and w <= ti_w + + def run_tile(tile_index, tile_range): + y_idx, y_end, x_idx, x_end = tile_range + tile_x = x[:, :, :, y_idx:y_end, x_idx:x_end] + tile_out = run_temporal_chunks(tile_x) + return tile_index, y_idx, y_end, x_idx, x_end, tile_out + + ordered_tile_outputs = ( + run_tile(tile_index, tile_range) + for tile_index, tile_range in enumerate(tile_ranges) + ) + + for _, y_idx, y_end, x_idx, x_end, tile_out in ordered_tile_outputs: + + if single_spatial_tile: + result = tile_out[:, :, :target_d, :target_h, :target_w] + if result.device != x.device or result.dtype != x.dtype: + result = result.to(device=x.device, dtype=x.dtype) + if x.shape[2] == 1 and sf_t == 1: + result = result.squeeze(2) + bar.update(1) + return result + + if result is None: + b_out, c_out = tile_out.shape[0], tile_out.shape[1] + result = torch.zeros((b_out, c_out, target_d, target_h, target_w), device=storage_device, dtype=torch.float32) + count = torch.zeros((1, 1, 1, target_h, target_w), device=storage_device, dtype=torch.float32) + + if encode: + ys, ye = y_idx // sf_s, (y_idx // sf_s) + tile_out.shape[3] + xs, xe = x_idx // sf_s, (x_idx // sf_s) + tile_out.shape[4] + cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2)) + cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2)) + else: + ys, ye = y_idx * sf_s, (y_idx * sf_s) + tile_out.shape[3] + xs, xe = x_idx * sf_s, (x_idx * sf_s) + tile_out.shape[4] + cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2)) + cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2)) + + w_h = torch.ones((tile_out.shape[3],), device=storage_device) + w_w = torch.ones((tile_out.shape[4],), device=storage_device) + + if cur_ov_h > 0: + r = get_ramp(cur_ov_h) + if y_idx > 0: + w_h[:cur_ov_h] = r + if y_end < h: + w_h[-cur_ov_h:] = 1.0 - r + + if cur_ov_w > 0: + r = get_ramp(cur_ov_w) + if x_idx > 0: + w_w[:cur_ov_w] = r + if x_end < w: + w_w[-cur_ov_w:] = 1.0 - r + + final_weight = w_h.view(1,1,1,-1,1) * w_w.view(1,1,1,1,-1) + + valid_d = min(tile_out.shape[2], result.shape[2]) + tile_out = tile_out[:, :, :valid_d, :, :] + + tile_out.mul_(final_weight) + + result[:, :, :valid_d, ys:ye, xs:xe] += tile_out + count[:, :, :, ys:ye, xs:xe] += final_weight + + del tile_out, final_weight, w_h, w_w + bar.update(1) + + result.div_(count.clamp(min=1e-6)) + + if result.device != x.device or result.dtype != x.dtype: + result = result.to(device=x.device, dtype=x.dtype) + + if x.shape[2] == 1 and sf_t == 1: + result = result.squeeze(2) + + return result + +_NORM_LIMIT = float("inf") +def get_norm_limit(): + return _NORM_LIMIT + + +def set_norm_limit(value: Optional[float] = None): + global _NORM_LIMIT + if value is None: + value = float("inf") + _NORM_LIMIT = value + +@contextmanager +def ignore_padding(model): + orig_padding = model.padding + model.padding = (0, 0, 0) + try: + yield + finally: + model.padding = orig_padding + +class MemoryState(Enum): + DISABLED = 0 + INITIALIZING = 1 + ACTIVE = 2 + UNSET = 3 + +def get_cache_size(conv_module, input_len, pad_len, dim=0): + dilated_kernel_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1 + output_len = (input_len + pad_len - dilated_kernel_size) // conv_module.stride[dim] + 1 + remain_len = ( + input_len + pad_len - ((output_len - 1) * conv_module.stride[dim] + dilated_kernel_size) + ) + overlap_len = dilated_kernel_size - conv_module.stride[dim] + cache_len = overlap_len + remain_len + + if output_len <= 0: + raise ValueError( + f"SeedVR2 VAE cache input is too short for convolution: input_len={input_len}, pad_len={pad_len}." + ) + return cache_len + +class DiagonalGaussianDistribution(object): + def __init__(self, parameters: torch.Tensor): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, BYTEDANCE_LOGVAR_CLAMP_MIN, BYTEDANCE_LOGVAR_CLAMP_MAX) + + def mode(self): + return self.mean + +class SpatialNorm(nn.Module): + def __init__( + self, + f_channels: int, + zq_channels: int, + ): + super().__init__() + self.norm_layer = ops.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True) + self.conv_y = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0) + self.conv_b = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, f: torch.Tensor, zq: torch.Tensor) -> torch.Tensor: + f_size = f.shape[-2:] + zq = F.interpolate(zq, size=f_size, mode="nearest") + norm_f = self.norm_layer(f) + new_f = norm_f * self.conv_y(zq) + self.conv_b(zq) + return new_f + +class Attention(nn.Module): + def __init__( + self, + query_dim: int, + heads: int = 8, + dim_head: int = 64, + bias: bool = False, + norm_num_groups: Optional[int] = None, + spatial_norm_dim: Optional[int] = None, + out_bias: bool = True, + eps: float = 1e-5, + rescale_output_factor: float = 1.0, + residual_connection: bool = False, + ): + super().__init__() + + self.inner_dim = dim_head * heads + self.rescale_output_factor = rescale_output_factor + self.residual_connection = residual_connection + self.out_dim = query_dim + self.heads = heads + + if norm_num_groups is not None: + self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True) + else: + self.group_norm = None + + if spatial_norm_dim is not None: + self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim) + else: + self.spatial_norm = None + + self.to_q = ops.Linear(query_dim, self.inner_dim, bias=bias) + self.to_k = ops.Linear(query_dim, self.inner_dim, bias=bias) + self.to_v = ops.Linear(query_dim, self.inner_dim, bias=bias) + self.to_out = nn.ModuleList([]) + self.to_out.append(ops.Linear(self.inner_dim, self.out_dim, bias=out_bias)) + self.to_out.append(nn.Identity()) + + self.optimized_vae_attention = vae_attention() + + def forward( + self, + hidden_states: torch.Tensor, + temb: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + + residual = hidden_states + if self.spatial_norm is not None: + hidden_states = self.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size = hidden_states.shape[0] + + if self.group_norm is not None: + hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = self.to_q(hidden_states) + key = self.to_k(hidden_states) + value = self.to_v(hidden_states) + + inner_dim = key.shape[-1] + head_dim = inner_dim // self.heads + + query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) + + key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) + + if input_ndim == 4 and self.heads == 1: + query = query.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width) + key = key.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width) + value = value.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width) + hidden_states = self.optimized_vae_attention(query, key, value).reshape(batch_size, self.heads, head_dim, height * width).transpose(2, 3) + else: + hidden_states = optimized_attention(query, key, value, heads = self.heads, skip_reshape=True, skip_output_reshape=True) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + hidden_states = self.to_out[0](hidden_states) + hidden_states = self.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if self.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / self.rescale_output_factor + + return hidden_states + + +def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor: + input_dtype = x.dtype + if isinstance(norm_layer, (nn.LayerNorm, nn.RMSNorm)): + if x.ndim == 4: + x = x.permute(0, 2, 3, 1) + x = norm_layer(x) + x = x.permute(0, 3, 1, 2) + return x.to(input_dtype) + if x.ndim == 5: + x = x.permute(0, 2, 3, 4, 1) + x = norm_layer(x) + x = x.permute(0, 4, 1, 2, 3) + return x.to(input_dtype) + if isinstance(norm_layer, (nn.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)): + if x.ndim <= 4: + return norm_layer(x).to(input_dtype) + if x.ndim == 5: + b, c, t, h, w = x.shape + x = x.transpose(1, 2).reshape(b * t, c, h, w) + memory_occupy = x.numel() * x.element_size() / 1024**3 + if isinstance(norm_layer, nn.GroupNorm) and memory_occupy > get_norm_limit(): + num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups) + if norm_layer.num_groups % num_chunks != 0: + raise ValueError( + f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}." + ) + num_groups_per_chunk = norm_layer.num_groups // num_chunks + + weights = comfy.ops.cast_to_input(norm_layer.weight, x).chunk(num_chunks, dim=0) + biases = comfy.ops.cast_to_input(norm_layer.bias, x).chunk(num_chunks, dim=0) + x = list(x.chunk(num_chunks, dim=1)) + for i, (w, bias) in enumerate(zip(weights, biases)): + x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps) + x[i] = x[i].to(input_dtype) + x = torch.cat(x, dim=1) + else: + x = norm_layer(x) + x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2) + return x.to(input_dtype) + raise TypeError(f"SeedVR2 VAE unsupported norm layer type: {type(norm_layer).__name__}") + +_receptive_field_t = Literal["half", "full"] + +def extend_head(tensor, times: int = 2, memory = None): + if memory is not None: + return torch.cat((memory.to(tensor), tensor), dim=2) + if times < 0: + raise ValueError(f"SeedVR2 VAE extend_head expected times >= 0, got {times}.") + if times == 0: + return tensor + else: + tile_repeat = [1] * tensor.ndim + tile_repeat[2] = times + return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2) + +def cache_send_recv(tensor, cache_size, times, memory=None): + recv_buffer = None + + if memory is not None: + recv_buffer = memory.to(tensor[0]) + elif times > 0: + tile_repeat = [1] * tensor[0].ndim + tile_repeat[2] = times + recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat) + + return recv_buffer + +class InflatedCausalConv3d(ops.Conv3d): + def __init__( + self, + *args, + inflation_mode, + **kwargs, + ): + self.inflation_mode = inflation_mode + super().__init__(*args, **kwargs) + self.temporal_padding = self.padding[0] + self.padding = (0, *self.padding[1:]) + self.memory_limit = float("inf") + self.logged_once = False + + def set_memory_limit(self, value: float): + self.memory_limit = value + + def _conv_forward(self, input, weight, bias, *args, **kwargs): + try: + return super()._conv_forward(input, weight, bias, *args, **kwargs) + except NotImplementedError: + # for: Could not run 'aten::cudnn_convolution' with arguments from the 'CPU' backend + if not self.logged_once: + logging.warning("VAE is on CPU for decoding. This is most likely due to not enough memory") + self.logged_once = True + return F.conv3d(input, weight, bias, *args, **kwargs) + + def memory_limit_conv( + self, + x, + *, + split_dim=3, + padding=(0, 0, 0, 0, 0, 0), + prev_cache=None, + ): + if math.isinf(self.memory_limit): + if prev_cache is not None: + x = torch.cat([prev_cache, x], dim=split_dim - 1) + return super().forward(x) + + shape = list(x.size()) + if prev_cache is not None: + shape[split_dim - 1] += prev_cache.size(split_dim - 1) + for i, pad_sum in enumerate((padding[4] + padding[5], padding[2] + padding[3], padding[0] + padding[1])): + shape[-3 + i] += pad_sum + memory_occupy = math.prod(shape) * x.element_size() / 1024**3 # GiB + if memory_occupy < self.memory_limit or split_dim == x.ndim: + x_concat = x + if prev_cache is not None: + x_concat = torch.cat([prev_cache, x], dim=split_dim - 1) + + def pad_and_forward(): + padded = F.pad(x_concat, padding, mode='constant', value=0.0) + if not padded.is_contiguous(): + padded = padded.contiguous() + with ignore_padding(self): + return torch.nn.Conv3d.forward(self, padded) + + return pad_and_forward() + + num_splits = math.ceil(memory_occupy / self.memory_limit) + size_per_split = x.size(split_dim) // num_splits + split_sizes = [size_per_split] * (num_splits - 1) + split_sizes += [x.size(split_dim) - sum(split_sizes)] + + x = list(x.split(split_sizes, dim=split_dim)) + if prev_cache is not None: + prev_cache = list(prev_cache.split(split_sizes, dim=split_dim)) + cache = None + for idx in range(len(x)): + if prev_cache is not None: + x[idx] = torch.cat([prev_cache[idx], x[idx]], dim=split_dim - 1) + + lpad_dim = (x[idx].ndim - split_dim - 1) * 2 + rpad_dim = lpad_dim + 1 + padding = list(padding) + padding[lpad_dim] = self.padding[split_dim - 2] if idx == 0 else 0 + padding[rpad_dim] = self.padding[split_dim - 2] if idx == len(x) - 1 else 0 + pad_len = padding[lpad_dim] + padding[rpad_dim] + padding = tuple(padding) + + next_cache = None + cache_len = cache.size(split_dim) if cache is not None else 0 + next_cache_size = get_cache_size( + conv_module=self, + input_len=x[idx].size(split_dim) + cache_len, + pad_len=pad_len, + dim=split_dim - 2, + ) + if next_cache_size != 0: + if next_cache_size > x[idx].size(split_dim): + raise ValueError( + f"SeedVR2 VAE cache size {next_cache_size} exceeds split size {x[idx].size(split_dim)}." + ) + next_cache = ( + x[idx].transpose(0, split_dim)[-next_cache_size:].transpose(0, split_dim) + ) + + x[idx] = self.memory_limit_conv( + x[idx], + split_dim=split_dim + 1, + padding=padding, + prev_cache=cache + ) + + cache = next_cache + + output = torch.cat(x, dim=split_dim) + return output + + def forward( + self, + input, + memory_state: MemoryState = MemoryState.UNSET, + memory_cache = None, + ) -> Tensor: + if memory_state == MemoryState.UNSET: + raise ValueError("SeedVR2 VAE convolution requires an explicit MemoryState.") + if memory_cache is None: + memory_cache = {} + if memory_state != MemoryState.ACTIVE: + memory_cache.pop(self, None) + if ( + math.isinf(self.memory_limit) + and torch.is_tensor(input) + ): + return self.basic_forward(input, memory_state, memory_cache) + return self.slicing_forward(input, memory_state, memory_cache) + + def basic_forward(self, input: Tensor, memory_state: MemoryState = MemoryState.UNSET, memory_cache = None): + mem_size = self.stride[0] - self.kernel_size[0] + memory = memory_cache.get(self) if memory_cache is not None else None + if (memory is not None) and (memory_state == MemoryState.ACTIVE): + input = extend_head(input, memory=memory, times=-1) + else: + input = extend_head(input, times=self.temporal_padding * 2) + next_memory = ( + input[:, :, mem_size:].detach() + if (mem_size != 0 and memory_state != MemoryState.DISABLED) + else None + ) + if memory_cache is not None and memory_state != MemoryState.DISABLED: + if next_memory is None: + memory_cache.pop(self, None) + else: + memory_cache[self] = next_memory + return super().forward(input) + + def slicing_forward( + self, + input, + memory_state: MemoryState = MemoryState.UNSET, + memory_cache = None, + ) -> Tensor: + if memory_cache is None: + memory_cache = {} + squeeze_out = False + if torch.is_tensor(input): + input = [input] + squeeze_out = True + + cache_size = self.kernel_size[0] - self.stride[0] + memory = memory_cache.get(self) if memory_cache is not None else None + cache = cache_send_recv( + input, cache_size=cache_size, memory=memory, times=self.temporal_padding * 2 + ) + + if ( + memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE] + and cache_size != 0 + ): + if cache_size > input[-1].size(2) and cache is not None and len(input) == 1: + input[0] = torch.cat([cache, input[0]], dim=2) + cache = None + if cache_size <= input[-1].size(2): + memory_cache[self] = input[-1][:, :, -cache_size:].detach().contiguous() + + padding = tuple(x for x in reversed(self.padding) for _ in range(2)) + for i in range(len(input)): + next_cache = None + cache_size = 0 + if i < len(input) - 1: + cache_len = cache.size(2) if cache is not None else 0 + cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0) + if cache_size != 0: + if cache_size > input[i].size(2) and cache is not None: + input[i] = torch.cat([cache, input[i]], dim=2) + cache = None + if cache_size > input[i].size(2): + raise ValueError(f"SeedVR2 VAE cache size {cache_size} exceeds input length {input[i].size(2)}.") + next_cache = input[i][:, :, -cache_size:] + + input[i] = self.memory_limit_conv( + input[i], + padding=padding, + prev_cache=cache + ) + + cache = next_cache + + return input[0] if squeeze_out else input + +def remove_head(tensor: Tensor, times: int = 1) -> Tensor: + if times == 0: + return tensor + return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2) + +class Upsample3D(nn.Module): + + def __init__( + self, + channels, + out_channels = None, + inflation_mode = "tail", + temporal_up: bool = False, + spatial_up: bool = True, + ): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + + conv = InflatedCausalConv3d( + self.channels, + self.out_channels, + 3, + padding=1, + inflation_mode=inflation_mode, + ) + + self.temporal_up = temporal_up + self.spatial_up = spatial_up + self.temporal_ratio = 2 if temporal_up else 1 + self.spatial_ratio = 2 if spatial_up else 1 + + upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio + self.upscale_conv = ops.Conv3d( + self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0 + ) + + self.conv = conv + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state=None, + memory_cache=None, + ) -> torch.FloatTensor: + if hidden_states.shape[1] != self.channels: + raise ValueError(f"SeedVR2 upsample expected {self.channels} channels, got {hidden_states.shape[1]}.") + + hidden_states = self.upscale_conv(hidden_states) + b, channels, f, h, w = hidden_states.shape + c = channels // (self.spatial_ratio * self.spatial_ratio * self.temporal_ratio) + hidden_states = hidden_states.view(b, self.spatial_ratio, self.spatial_ratio, self.temporal_ratio, c, f, h, w) + hidden_states = hidden_states.permute(0, 4, 5, 3, 6, 1, 7, 2).reshape( + b, + c, + f * self.temporal_ratio, + h * self.spatial_ratio, + w * self.spatial_ratio, + ) + + if self.temporal_up and memory_state != MemoryState.ACTIVE: + hidden_states = remove_head(hidden_states) + + hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache) + + return hidden_states + + +class Downsample3D(nn.Module): + def __init__( + self, + channels, + out_channels = None, + inflation_mode = "tail", + spatial_down: bool = False, + temporal_down: bool = False, + ): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.temporal_down = temporal_down + self.spatial_down = spatial_down + + self.temporal_ratio = 2 if temporal_down else 1 + self.spatial_ratio = 2 if spatial_down else 1 + + self.temporal_kernel = 3 if temporal_down else 1 + self.spatial_kernel = 3 if spatial_down else 1 + + self.conv = InflatedCausalConv3d( + self.channels, + self.out_channels, + kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel), + stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio), + padding=(1 if self.temporal_down else 0, 0, 0), + inflation_mode=inflation_mode, + ) + + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state = None, + memory_cache = None, + ) -> torch.FloatTensor: + + if hidden_states.shape[1] != self.channels: + raise ValueError(f"SeedVR2 downsample expected {self.channels} channels, got {hidden_states.shape[1]}.") + + if self.spatial_down: + pad = (0, 1, 0, 1) + hidden_states = F.pad(hidden_states, pad, mode="constant", value=0) + + if hidden_states.shape[1] != self.channels: + raise ValueError(f"SeedVR2 downsample expected {self.channels} channels after padding, got {hidden_states.shape[1]}.") + + hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache) + + return hidden_states + + +class ResnetBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + temb_channels: int = 512, + groups: int = 32, + groups_out: Optional[int] = None, + eps: float = 1e-6, + output_scale_factor: float = 1.0, + skip_time_act: bool = False, + inflation_mode = "tail", + time_receptive_field: _receptive_field_t = "half", + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = in_channels if out_channels is None else out_channels + self.output_scale_factor = output_scale_factor + self.skip_time_act = skip_time_act + self.nonlinearity = nn.SiLU() + if temb_channels is not None: + self.time_emb_proj = ops.Linear(temb_channels, self.out_channels) + else: + self.time_emb_proj = None + self.norm1 = ops.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True) + if groups_out is None: + groups_out = groups + self.norm2 = ops.GroupNorm(num_groups=groups_out, num_channels=self.out_channels, eps=eps, affine=True) + self.use_in_shortcut = self.in_channels != self.out_channels + self.conv1 = InflatedCausalConv3d( + self.in_channels, + self.out_channels, + kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3), + stride=1, + padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1), + inflation_mode=inflation_mode, + ) + + self.conv2 = InflatedCausalConv3d( + self.out_channels, + self.out_channels, + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.conv_shortcut = None + if self.use_in_shortcut: + self.conv_shortcut = InflatedCausalConv3d( + self.in_channels, + self.out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=True, + inflation_mode=inflation_mode, + ) + + def forward(self, input_tensor, temb, memory_state = None, memory_cache = None): + hidden_states = input_tensor + + hidden_states = causal_norm_wrapper(self.norm1, hidden_states) + + hidden_states = self.nonlinearity(hidden_states) + + hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache) + + if self.time_emb_proj is not None: + if not self.skip_time_act: + temb = self.nonlinearity(temb) + temb = self.time_emb_proj(temb)[:, :, None, None] + + if temb is not None: + hidden_states = hidden_states + temb + + hidden_states = causal_norm_wrapper(self.norm2, hidden_states) + + hidden_states = self.nonlinearity(hidden_states) + + hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state, memory_cache=memory_cache) + + output_tensor = (input_tensor + hidden_states) / self.output_scale_factor + + return output_tensor + + +class DownEncoderBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_groups: int = 32, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + inflation_mode = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_down: bool = True, + spatial_down: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock3D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + output_scale_factor=output_scale_factor, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample3D( + out_channels, + out_channels=out_channels, + temporal_down=temporal_down, + spatial_down=spatial_down, + inflation_mode=inflation_mode, + ) + ] + ) + else: + self.downsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state = None, + memory_cache = None, + ) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache) + + return hidden_states + + +class UpDecoderBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_groups: int = 32, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temb_channels: Optional[int] = None, + inflation_mode = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_up: bool = True, + spatial_up: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock3D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + output_scale_factor=output_scale_factor, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + Upsample3D( + out_channels, + out_channels=out_channels, + temporal_up=temporal_up, + spatial_up=spatial_up, + inflation_mode=inflation_mode, + ) + ] + ) + else: + self.upsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + memory_state=None, + memory_cache=None, + ) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache) + + return hidden_states + + +class UNetMidBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_groups: int = 32, + add_attention: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + inflation_mode = "tail", + time_receptive_field: _receptive_field_t = "half", + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + self.add_attention = add_attention + + resnets = [ + ResnetBlock3D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + output_scale_factor=output_scale_factor, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ] + attentions = [] + + if attention_head_dim is None: + attention_head_dim = in_channels + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=( + resnet_groups if resnet_time_scale_shift == "default" else None + ), + spatial_norm_dim=( + temb_channels if resnet_time_scale_shift == "spatial" else None + ), + residual_connection=True, + bias=True, + ) + ) + else: + attentions.append(None) + + resnets.append( + ResnetBlock3D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + output_scale_factor=output_scale_factor, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states, temb=None, memory_state=None, memory_cache=None): + video_length = hidden_states.size(2) + hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + b, c, f, h, w = hidden_states.shape + hidden_states = hidden_states.transpose(1, 2).reshape(b * f, c, h, w) + hidden_states = attn(hidden_states, temb=temb) + hidden_states = hidden_states.reshape(b, video_length, c, h, w).transpose(1, 2) + hidden_states = resnet(hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache) + + return hidden_states + + +class Encoder3D(nn.Module): + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + mid_block_add_attention=True, + temporal_down_num: int = 2, + inflation_mode = "tail", + time_receptive_field: _receptive_field_t = "half", + ): + super().__init__() + self.layers_per_block = layers_per_block + self.temporal_down_num = temporal_down_num + + self.conv_in = InflatedCausalConv3d( + in_channels, + block_out_channels[0], + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.mid_block = None + self.down_blocks = nn.ModuleList([]) + + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1 + + if down_block_type != "DownEncoderBlock3D": + raise ValueError(f"SeedVR2 encoder only supports DownEncoderBlock3D, got {down_block_type}.") + + down_block = DownEncoderBlock3D( + num_layers=self.layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=not is_final_block, + resnet_eps=1e-6, + resnet_groups=norm_num_groups, + temporal_down=is_temporal_down_block, + spatial_down=True, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + self.down_blocks.append(down_block) + + self.mid_block = UNetMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + output_scale_factor=1, + resnet_time_scale_shift="default", + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=None, + add_attention=mid_block_add_attention, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + self.conv_norm_out = ops.GroupNorm( + num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6 + ) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels + self.conv_out = InflatedCausalConv3d( + block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode + ) + + + def forward( + self, + sample: torch.FloatTensor, + memory_state = None, + memory_cache = None, + ) -> torch.FloatTensor: + sample = sample.to(next(self.parameters()).device) + sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache) + for down_block in self.down_blocks: + sample = down_block(sample, memory_state=memory_state, memory_cache=memory_cache) + + sample = self.mid_block(sample, memory_state=memory_state, memory_cache=memory_cache) + + sample = causal_norm_wrapper(self.conv_norm_out, sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache) + + return sample + + +class Decoder3D(nn.Module): + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + mid_block_add_attention=True, + inflation_mode = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_up_num: int = 2, + ): + super().__init__() + self.layers_per_block = layers_per_block + self.temporal_up_num = temporal_up_num + + self.conv_in = InflatedCausalConv3d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + temb_channels = None + + self.mid_block = UNetMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + output_scale_factor=1, + resnet_time_scale_shift="default", + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + add_attention=mid_block_add_attention, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + is_temporal_up_block = i < self.temporal_up_num + if up_block_type != "UpDecoderBlock3D": + raise ValueError(f"SeedVR2 decoder only supports UpDecoderBlock3D, got {up_block_type}.") + up_block = UpDecoderBlock3D( + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + temporal_up=is_temporal_up_block, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + self.conv_norm_out = ops.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6 + ) + self.conv_act = nn.SiLU() + self.conv_out = InflatedCausalConv3d( + block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode + ) + + + def forward( + self, + sample: torch.FloatTensor, + latent_embeds: Optional[torch.FloatTensor] = None, + memory_state = None, + memory_cache = None, + ) -> torch.FloatTensor: + + sample = sample.to(next(self.parameters()).device) + sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + sample = self.mid_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache) + sample = sample.to(upscale_dtype) + + for up_block in self.up_blocks: + sample = up_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache) + + sample = causal_norm_wrapper(self.conv_norm_out, sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache) + + return sample + +class VideoAutoencoderKL(nn.Module): + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + layers_per_block: int = 2, + latent_channels: int = SEEDVR2_LATENT_CHANNELS, + norm_num_groups: int = 32, + temporal_scale_num: int = 2, + inflation_mode = "pad", + time_receptive_field: _receptive_field_t = "full", + slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN, + ): + self.slicing_sample_min_size = slicing_sample_min_size + self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num) + block_out_channels = BYTEDANCE_BLOCK_OUT_CHANNELS + down_block_types = ("DownEncoderBlock3D",) * 4 + up_block_types = ("UpDecoderBlock3D",) * 4 + super().__init__() + + self.encoder = Encoder3D( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + temporal_down_num=temporal_scale_num, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + self.decoder = Decoder3D( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + temporal_up_num=temporal_scale_num, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + self.use_slicing = True + + def encode(self, x: torch.FloatTensor, return_dict: bool = True): + h = self.slicing_encode(x) + posterior = DiagonalGaussianDistribution(h).mode() + + if not return_dict: + return (posterior,) + + return posterior + + def decode_( + self, z: torch.Tensor, return_dict: bool = True + ): + decoded = self.slicing_decode(z) + + if not return_dict: + return (decoded,) + + return decoded + + def _encode( + self, x, memory_state = MemoryState.DISABLED, memory_cache = None + ) -> torch.Tensor: + _x = x.to(self.device) + h = self.encoder(_x, memory_state=memory_state, memory_cache=memory_cache) + return h.to(x.device) + + def _decode( + self, z, memory_state = MemoryState.DISABLED, memory_cache = None + ) -> torch.Tensor: + _z = z.to(self.device) + output = self.decoder(_z, memory_state=memory_state, memory_cache=memory_cache) + return output.to(z.device) + + def slicing_encode(self, x: torch.Tensor) -> torch.Tensor: + if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size: + memory_cache = {} + split_size = max( + self.slicing_sample_min_size, + getattr(self, "temporal_downsample_factor", 1), + ) + x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2)) + min_active_len = getattr(self, "temporal_downsample_factor", 1) + if len(x_slices) > 1 and x_slices[-1].shape[2] < min_active_len: + x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2) + x_slices.pop() + encoded_slices = [ + self._encode( + torch.cat((x[:, :, :1], x_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, + memory_cache=memory_cache, + ) + ] + for x_idx in range(1, len(x_slices)): + encoded_slices.append( + self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache) + ) + out = torch.cat(encoded_slices, dim=2) + return out + else: + return self._encode(x) + + def slicing_decode(self, z: torch.Tensor) -> torch.Tensor: + if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size: + memory_cache = {} + z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2) + decoded_slices = [ + self._decode( + torch.cat((z[:, :, :1], z_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, + memory_cache=memory_cache, + ) + ] + for z_idx in range(1, len(z_slices)): + decoded_slices.append( + self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache) + ) + out = torch.cat(decoded_slices, dim=2) + return out + else: + return self._decode(z) + + def forward(self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all"): + def _unwrap(value): + return value[0] if isinstance(value, tuple) else value + + if mode == "encode": + return _unwrap(self.encode(x)) + if mode == "decode": + return _unwrap(self.decode_(x)) + if mode == "all": + latent = _unwrap(self.encode(x)) + return _unwrap(self.decode_(latent)) + raise ValueError(f"Unknown SeedVR2 VAE forward mode: {mode}") + +class VideoAutoencoderKLWrapper(VideoAutoencoderKL): + def __init__( + self, + spatial_downsample_factor = 8, + temporal_downsample_factor = 4, + ): + self.spatial_downsample_factor = spatial_downsample_factor + self.temporal_downsample_factor = temporal_downsample_factor + super().__init__() + self.set_memory_limit(BYTEDANCE_VAE_CONV_MEM_GIB, BYTEDANCE_VAE_NORM_MEM_GIB) + + def forward(self, x: torch.FloatTensor): + z, p = self._encode_with_raw_latent(x) + x = self.decode(z) + return x, z, p + + def _encode_with_raw_latent(self, x): + if x.ndim == 4: + x = x.unsqueeze(2) + self.device = x.device + p = super().encode(x) + z = p.squeeze(2) + return z, p + + def encode(self, x): + z, _ = self._encode_with_raw_latent(x) + return z + + def decode(self, z, seedvr2_tiling=None): + seedvr2_tiling = {} if seedvr2_tiling is None else seedvr2_tiling + if not isinstance(seedvr2_tiling, dict): + raise RuntimeError( + "SeedVR2 VideoAutoencoderKLWrapper.decode: `seedvr2_tiling` must be a dict; " + f"got {type(seedvr2_tiling).__name__} with value {seedvr2_tiling!r}." + ) + + if z.ndim == 5: + _, c, _, _, _ = z.shape + if c != SEEDVR2_LATENT_CHANNELS: + raise RuntimeError( + "SeedVR2 VideoAutoencoderKLWrapper.decode: 5-D latent input must " + f"have {SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(z.shape)}." + ) + latent = z + elif z.ndim == 4: + b, tc, h, w = z.shape + if tc % SEEDVR2_LATENT_CHANNELS != 0: + raise RuntimeError( + "SeedVR2 VideoAutoencoderKLWrapper.decode: 4-D latent input must " + f"use collapsed channel layout (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W); " + f"got shape {tuple(z.shape)}." + ) + latent = z.reshape(b, SEEDVR2_LATENT_CHANNELS, -1, h, w) + else: + raise RuntimeError( + "SeedVR2 VideoAutoencoderKLWrapper.decode: latent input must be " + f"4-D collapsed (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W) or " + f"5-D (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); " + f"got shape {tuple(z.shape)}." + ) + scale = BYTEDANCE_VAE_SCALING_FACTOR + shift = BYTEDANCE_VAE_SHIFTING_FACTOR + latent = latent / scale + shift + + self.device = latent.device + enable_tiling = seedvr2_tiling.get("enable_tiling", False) + + if enable_tiling: + decode_seedvr2_args = dict(seedvr2_tiling) + decode_seedvr2_args.pop("enable_tiling", None) + tile_h, tile_w = decode_seedvr2_args.get("tile_size", (512, 512)) + ov_h, ov_w = decode_seedvr2_args.get("tile_overlap", (64, 64)) + decode_seedvr2_args["tile_overlap"] = ( + min(ov_h, max(0, tile_h - 8)), + min(ov_w, max(0, tile_w - 8)), + ) + x = tiled_vae(latent, self, **decode_seedvr2_args, encode=False) + if x.ndim == 4: + # tiled_vae squeezes the temporal axis when + # temporal_downsample_factor == 1 AND latent T == 1 + # (see tiled_vae line 179-180); re-add it so the post-decode + # pipeline can keep batch and time distinct on the tiled path. + x = x.unsqueeze(2) + else: + x = super().decode_(latent) + + h, w = x.shape[-2:] + w2 = w - (w % 2) + h2 = h - (h % 2) + x = x[..., :h2, :w2] + + return x + + def decode_tiled(self, z, tile_x=32, tile_y=32, overlap=8, tile_t=None, overlap_t=None): + # SeedVR2's causal VAE owns temporal via the MemoryState cache; external + # temporal tiling breaks that continuity, so only spatial tiling is applied. + sf = self.spatial_downsample_factor + seedvr2_tiling = { + "enable_tiling": True, + "tile_size": (tile_y * sf, tile_x * sf), + "tile_overlap": (overlap * sf, overlap * sf), + "temporal_size": None, + "temporal_overlap": None, + } + return self.decode(z, seedvr2_tiling=seedvr2_tiling) + + def encode_tiled(self, x, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None): + # External temporal tiling knobs are discarded; the causal VAE keeps its + # own internal MemoryState slicing. + if tile_y is None: + tile_y = 512 + if tile_x is None: + tile_x = 512 + if overlap is None: + overlap_y = 64 + overlap_x = 64 + else: + overlap_y = overlap + overlap_x = overlap + overlap_y = min(overlap_y, max(0, tile_y - 8)) + overlap_x = min(overlap_x, max(0, tile_x - 8)) + self.device = x.device + return tiled_vae( + x, + self, + tile_size=(tile_y, tile_x), + tile_overlap=(overlap_y, overlap_x), + temporal_size=None, + temporal_overlap=None, + encode=True, + ) + + def comfy_format_encoded(self, samples): + if samples.ndim == 4: + samples = samples.unsqueeze(2) + samples = samples.contiguous() + samples = samples * BYTEDANCE_VAE_SCALING_FACTOR + return samples + + def comfy_memory_used_decode(self, shape): + bytes_per_output_pixel = 160 + + def output_pixels(latent_t, latent_h, latent_w): + output_t = max(1, (latent_t - 1) * 4 + 1) + return output_t * latent_h * 8 * latent_w * 8 + + # SeedVR2 decode performs full-frame LAB histogram matching: fp32 channels + # plus int64 sort indices dominate peak memory, not the VAE weight dtype. + if len(shape) == 5: + candidates = [] + if shape[1] == SEEDVR2_LATENT_CHANNELS: + candidates.append((shape[2], shape[3], shape[4])) + if shape[-1] == SEEDVR2_LATENT_CHANNELS: + candidates.append((shape[1], shape[2], shape[3])) + if len(candidates) == 0: + candidates.append((shape[2], shape[3], shape[4])) + pixels = max(output_pixels(*candidate) for candidate in candidates) + elif len(shape) == 4: + latent_t = max(1, (shape[1] + SEEDVR2_LATENT_CHANNELS - 1) // SEEDVR2_LATENT_CHANNELS) + pixels = output_pixels(latent_t, shape[2], shape[3]) + else: + pixels = output_pixels(1, shape[-2], shape[-1]) + return pixels * bytes_per_output_pixel + + def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]): + set_norm_limit(norm_max_mem) + for m in self.modules(): + if isinstance(m, InflatedCausalConv3d): + m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf")) diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py index 70dfe7b16..1c9782a38 100644 --- a/comfy/ldm/wan/model.py +++ b/comfy/ldm/wan/model.py @@ -8,7 +8,7 @@ from einops import rearrange from comfy.ldm.modules.attention import optimized_attention from comfy.ldm.flux.layers import EmbedND -from comfy.ldm.flux.math import apply_rope1 +from comfy.ldm.flux.math import apply_rope1, rope import comfy.ldm.common_dit import comfy.model_management import comfy.patcher_extension @@ -570,6 +570,14 @@ class WanModel(torch.nn.Module): full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2) x = torch.concat((full_ref, x), dim=1) + # In-context reference (Bernini) + context_latents = kwargs.get("context_latents", None) + main_len = x.shape[1] + if context_latents is not None: + for lat in context_latents: + cl = self.patch_embedding(lat.float().to(x.device)).to(x.dtype).flatten(2).transpose(1, 2) + x = torch.cat([x, cl], dim=1) + # context context = self.text_embedding(context) @@ -599,6 +607,9 @@ class WanModel(torch.nn.Module): # head x = self.head(x, e) + if context_latents is not None: + x = x[:, :main_len] + if full_ref is not None: x = x[:, full_ref.shape[1]:] @@ -606,7 +617,7 @@ class WanModel(torch.nn.Module): x = self.unpatchify(x, grid_sizes) return x - def rope_encode(self, t, h, w, t_start=0, steps_t=None, steps_h=None, steps_w=None, device=None, dtype=None, transformer_options={}): + def rope_encode(self, t, h, w, t_start=0, steps_t=None, steps_h=None, steps_w=None, device=None, dtype=None, transformer_options={}, source_id=0): patch_size = self.patch_size t_len = ((t + (patch_size[0] // 2)) // patch_size[0]) h_len = ((h + (patch_size[1] // 2)) // patch_size[1]) @@ -638,6 +649,13 @@ class WanModel(torch.nn.Module): img_ids = img_ids.reshape(1, -1, img_ids.shape[-1]) freqs = self.rope_embedder(img_ids).movedim(1, 2) + + # In-context reference: a non-zero source_id composes an extra rotation into the spatial rope + if source_id: + d = self.dim // self.num_heads + pos = torch.tensor([[float(source_id)]], device=freqs.device, dtype=torch.float32) + id_rot = rope(pos, d, self.rope_embedder.theta).reshape(1, 1, 1, d // 2, 2, 2).to(freqs.dtype) + freqs = torch.einsum('...ij,...jk->...ik', freqs, id_rot) return freqs def forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, **kwargs): @@ -661,6 +679,15 @@ class WanModel(torch.nn.Module): t_len += 1 freqs = self.rope_encode(t_len, h, w, device=x.device, dtype=x.dtype, transformer_options=transformer_options) + + # In-context reference: one rope block per stream, each with it's own source_id (1, 2, ...) to distinguish from the target (id 0). + context_latents = kwargs.get("context_latents", None) + if context_latents is not None: + context_latents = [comfy.ldm.common_dit.pad_to_patch_size(lat, self.patch_size) for lat in context_latents] + for i, lat in enumerate(context_latents): + freqs = torch.cat([freqs, self.rope_encode(lat.shape[-3], lat.shape[-2], lat.shape[-1], device=x.device, dtype=x.dtype, transformer_options=transformer_options, source_id=i + 1)], dim=1) + kwargs = {**kwargs, "context_latents": context_latents} + return self.forward_orig(x, timestep, context, clip_fea=clip_fea, freqs=freqs, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w] def unpatchify(self, x, grid_sizes): @@ -1631,13 +1658,15 @@ class SCAILWanModel(WanModel): self.patch_embedding_pose = operations.Conv3d(in_dim, dim, kernel_size=patch_size, stride=patch_size, device=device, dtype=torch.float32) - def forward_orig(self, x, t, context, clip_fea=None, freqs=None, transformer_options={}, pose_latents=None, reference_latent=None, **kwargs): + def forward_orig(self, x, t, context, clip_fea=None, freqs=None, transformer_options={}, pose_latents=None, reference_latent=None, ref_mask_latents=None, sam_latents=None, **kwargs): if reference_latent is not None: x = torch.cat((reference_latent, x), dim=2) # embeddings x = self.patch_embedding(x.float()).to(x.dtype) + if ref_mask_latents is not None: # SCAIL-2 additive mask stream (one identity mask frame per reference, then video) + x = x + self.patch_embedding_mask(ref_mask_latents.float()).to(x.dtype) grid_sizes = x.shape[2:] transformer_options["grid_sizes"] = grid_sizes x = x.flatten(2).transpose(1, 2) @@ -1645,6 +1674,8 @@ class SCAILWanModel(WanModel): scail_pose_seq_len = 0 if pose_latents is not None: scail_x = self.patch_embedding_pose(pose_latents.float()).to(x.dtype) + if sam_latents is not None: # SCAIL-2 additive mask stream + scail_x = scail_x + self.patch_embedding_mask(sam_latents.float()).to(x.dtype) scail_x = scail_x.flatten(2).transpose(1, 2) scail_pose_seq_len = scail_x.shape[1] x = torch.cat([x, scail_x], dim=1) @@ -1695,16 +1726,44 @@ class SCAILWanModel(WanModel): return x - def rope_encode(self, t, h, w, t_start=0, steps_t=None, steps_h=None, steps_w=None, device=None, dtype=None, pose_latents=None, reference_latent=None, transformer_options={}): + # ref_mask_flag is a scalar bool (CONDConstant, SCAIL-2 only). False => replacement mode, + # which places ref/pose via H/W rope shifts instead of the animation-mode temporal offset. + # reference_latent may stack several frames: the last is the primary reference adjacent to the video, the earlier frames are additional references. + def rope_encode(self, t, h, w, t_start=0, steps_t=None, steps_h=None, steps_w=None, device=None, dtype=None, pose_latents=None, reference_latent=None, ref_mask_flag=None, transformer_options={}): + ref_t_patches = 0 + if reference_latent is not None: + ref_t_patches = (reference_latent.shape[2] + (self.patch_size[0] // 2)) // self.patch_size[0] + + if ref_mask_flag is not None and not bool(ref_mask_flag): + REF_ROPE_H = 120.0 + POSE_ROPE_W = 120.0 + + main_t_patches = t - ref_t_patches + video_t_start = max(ref_t_patches - 1, 0) + + parts = [] + if ref_t_patches > 0: + ref_tf = {"rope_options": {"shift_y": REF_ROPE_H, "shift_x": 0.0, "scale_y": 1.0, "scale_x": 1.0}} + parts.append(super().rope_encode(ref_t_patches, h, w, t_start=0, device=device, dtype=dtype, transformer_options=ref_tf)) + if main_t_patches > 0: + parts.append(super().rope_encode(main_t_patches, h, w, t_start=video_t_start, device=device, dtype=dtype, transformer_options=transformer_options)) + + if pose_latents is not None: + F_pose, H_pose, W_pose = pose_latents.shape[-3], pose_latents.shape[-2], pose_latents.shape[-1] + h_scale = h / H_pose + w_scale = w / W_pose + h_shift = (h_scale - 1) / 2 + w_shift = (w_scale - 1) / 2 + pose_tf = {"rope_options": {"shift_y": h_shift, "shift_x": POSE_ROPE_W + w_shift, "scale_y": h_scale, "scale_x": w_scale}} + parts.append(super().rope_encode(F_pose, H_pose, W_pose, t_start=video_t_start, device=device, dtype=dtype, transformer_options=pose_tf)) + + return torch.cat(parts, dim=1) + main_freqs = super().rope_encode(t, h, w, t_start=t_start, steps_t=steps_t, steps_h=steps_h, steps_w=steps_w, device=device, dtype=dtype, transformer_options=transformer_options) if pose_latents is None: return main_freqs - ref_t_patches = 0 - if reference_latent is not None: - ref_t_patches = (reference_latent.shape[2] + (self.patch_size[0] // 2)) // self.patch_size[0] - F_pose, H_pose, W_pose = pose_latents.shape[-3], pose_latents.shape[-2], pose_latents.shape[-1] # if pose is at half resolution, scale_y/scale_x=2 stretches the position range to cover the same RoPE extent as the main frames @@ -1719,12 +1778,16 @@ class SCAILWanModel(WanModel): return torch.cat([main_freqs, pose_freqs], dim=1) - def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, pose_latents=None, **kwargs): + def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, pose_latents=None, ref_mask_latents=None, sam_latents=None, **kwargs): bs, c, t, h, w = x.shape x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size) if pose_latents is not None: pose_latents = comfy.ldm.common_dit.pad_to_patch_size(pose_latents, self.patch_size) + if ref_mask_latents is not None: # SCAIL-2 + ref_mask_latents = comfy.ldm.common_dit.pad_to_patch_size(ref_mask_latents, self.patch_size) + if sam_latents is not None: # SCAIL-2 + sam_latents = comfy.ldm.common_dit.pad_to_patch_size(sam_latents, self.patch_size) t_len = t if time_dim_concat is not None: @@ -1737,5 +1800,15 @@ class SCAILWanModel(WanModel): reference_latent = comfy.ldm.common_dit.pad_to_patch_size(kwargs.pop("reference_latent"), self.patch_size) t_len += reference_latent.shape[2] - freqs = self.rope_encode(t_len, h, w, device=x.device, dtype=x.dtype, transformer_options=transformer_options, pose_latents=pose_latents, reference_latent=reference_latent) - return self.forward_orig(x, timestep, context, clip_fea=clip_fea, freqs=freqs, transformer_options=transformer_options, pose_latents=pose_latents, reference_latent=reference_latent, **kwargs)[:, :, :t, :h, :w] + ref_mask_flag = kwargs.pop("ref_mask_flag", None) # SCAIL-2 + + freqs = self.rope_encode(t_len, h, w, device=x.device, dtype=x.dtype, transformer_options=transformer_options, pose_latents=pose_latents, reference_latent=reference_latent, ref_mask_flag=ref_mask_flag) + return self.forward_orig(x, timestep, context, clip_fea=clip_fea, freqs=freqs, transformer_options=transformer_options, pose_latents=pose_latents, reference_latent=reference_latent, ref_mask_latents=ref_mask_latents, sam_latents=sam_latents, **kwargs)[:, :, :t, :h, :w] + + +class SCAIL2WanModel(SCAILWanModel): + """SCAIL-2: SCAIL-Preview + an additive binary multi-identity mask stream.""" + + def __init__(self, model_type="scail2", patch_size=(1, 2, 2), in_dim=20, mask_in_dim=28, dim=5120, operations=None, device=None, dtype=None, **kwargs): + super().__init__(model_type=model_type, patch_size=patch_size, in_dim=in_dim, dim=dim, operations=operations, device=device, dtype=dtype, **kwargs) + self.patch_embedding_mask = operations.Conv3d(mask_in_dim, dim, kernel_size=patch_size, stride=patch_size, device=device, dtype=torch.float32) diff --git a/comfy/lora.py b/comfy/lora.py index 4e0ea29e0..427cf98aa 100644 --- a/comfy/lora.py +++ b/comfy/lora.py @@ -326,6 +326,17 @@ def model_lora_keys_unet(model, key_map={}): key_map["transformer.{}".format(key_lora)] = k key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = k #SimpleTuner lycoris format + if isinstance(model, comfy.model_base.Krea2): + diffusers_keys = comfy.utils.krea2_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.") + for k in diffusers_keys: + if k.endswith(".weight"): + to = diffusers_keys[k] + key_lora = k[:-len(".weight")] + key_map["diffusion_model.{}".format(key_lora)] = to + key_map["transformer.{}".format(key_lora)] = to + key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = to + key_map[key_lora] = to + if isinstance(model, comfy.model_base.Lumina2): diffusers_keys = comfy.utils.z_image_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.") for k in diffusers_keys: @@ -357,6 +368,12 @@ def model_lora_keys_unet(model, key_map={}): key_lora = k[len("diffusion_model."):-len(".weight")] key_map["transformer.{}".format(key_lora)] = k + if isinstance(model, (comfy.model_base.LTXV, comfy.model_base.LTXAV)): + for k in sdk: + if k.startswith("diffusion_model.") and k.endswith(".weight"): + key_lora = k[len("diffusion_model."):-len(".weight")] + key_map["{}".format(key_lora)] = k + return key_map diff --git a/comfy/model_base.py b/comfy/model_base.py index 3e2d4e930..786a7c127 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -21,6 +21,7 @@ import comfy.ldm.hunyuan3dv2_1.hunyuandit import torch import logging import comfy.ldm.lightricks.av_model +import comfy.ldm.lightricks.symmetric_patchifier import comfy.context_windows from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep from comfy.ldm.cascade.stage_c import StageC @@ -54,7 +55,11 @@ import comfy.ldm.pixeldit.model import comfy.ldm.pixeldit.pid import comfy.ldm.ace.model import comfy.ldm.omnigen.omnigen2 +import comfy.ldm.seedvr.model +import comfy.ldm.boogu.model import comfy.ldm.qwen_image.model +import comfy.ldm.ideogram4.model +import comfy.ldm.krea2.model import comfy.ldm.kandinsky5.model import comfy.ldm.anima.model import comfy.ldm.ace.ace_step15 @@ -64,6 +69,7 @@ import comfy.ldm.ernie.model import comfy.ldm.sam3.detector import comfy.ldm.hidream_o1.model from comfy.ldm.hidream_o1.conditioning import build_extra_conds +import comfy.ldm.depth_anything_3.model import comfy.model_management import comfy.patcher_extension @@ -927,6 +933,17 @@ class HunyuanDiT(BaseModel): out['image_meta_size'] = comfy.conds.CONDRegular(torch.FloatTensor([[height, width, target_height, target_width, 0, 0]])) return out +class SeedVR2(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.seedvr.model.NaDiT) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + condition = kwargs.get("condition", None) + if condition is not None: + out["condition"] = comfy.conds.CONDRegular(condition) + return out + class PixArt(BaseModel): def __init__(self, model_config, model_type=ModelType.EPS, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.pixart.pixartms.PixArtMS) @@ -1201,6 +1218,127 @@ class LTXAV(BaseModel): def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs): return latent_image + def map_context_window_to_modalities(self, primary_indices, latent_shapes, dim): + result = [primary_indices] + if len(latent_shapes) < 2: + return result + + video_total = latent_shapes[0][dim] + + for i in range(1, len(latent_shapes)): + mod_total = latent_shapes[i][dim] + # Map each primary index to its proportional range of modality indices and + # concatenate in order. Preserves wrapped/strided geometry so the modality + # attends to the same temporal regions as the primary window. + mod_indices = [] + seen = set() + for v_idx in primary_indices: + a_start = min(int(round(v_idx * mod_total / video_total)), mod_total - 1) + a_end = min(int(round((v_idx + 1) * mod_total / video_total)), mod_total) + if a_end <= a_start: + a_end = a_start + 1 + for a in range(a_start, a_end): + if a not in seen: + seen.add(a) + mod_indices.append(a) + result.append(mod_indices) + + return result + + @staticmethod + def _get_guide_entries(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + entries = model_conds.get('guide_attention_entries') + if entries is not None and hasattr(entries, 'cond') and entries.cond: + return entries.cond + return None + + def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]): + # Audio denoise mask — slice using audio modality window + if cond_key == "audio_denoise_mask" and hasattr(window, 'modality_windows') and window.modality_windows: + audio_window = window.modality_windows.get(1) + if audio_window is not None and hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor): + sliced = audio_window.get_tensor(cond_value.cond, device, dim=2) + return cond_value._copy_with(sliced) + + # Video denoise mask — split into video + guide portions, slice each + if cond_key == "denoise_mask" and hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor): + cond_tensor = cond_value.cond + guide_count = cond_tensor.size(window.dim) - x_in.size(window.dim) + if guide_count > 0: + T_video = x_in.size(window.dim) + video_mask = cond_tensor.narrow(window.dim, 0, T_video) + guide_mask = cond_tensor.narrow(window.dim, T_video, guide_count) + sliced_video = window.get_tensor(video_mask, device, retain_index_list=retain_index_list) + suffix_indices = window.guide_frames_indices + if suffix_indices: + idx = tuple([slice(None)] * window.dim + [suffix_indices]) + sliced_guide = guide_mask[idx].to(device) + return cond_value._copy_with(torch.cat([sliced_video, sliced_guide], dim=window.dim)) + else: + return cond_value._copy_with(sliced_video) + + # Keyframe indices — regenerate pixel coords for window, select guide positions + if cond_key == "keyframe_idxs": + kf_local_pos = window.guide_kf_local_positions + if not kf_local_pos: + return cond_value._copy_with(cond_value.cond[:, :, :0, :]) # empty + H, W = x_in.shape[3], x_in.shape[4] + window_len = len(window.index_list) + # account for causal_window_fix anchor in coord space size + anchor_idx = getattr(window, 'causal_anchor_index', None) + if anchor_idx is not None and anchor_idx >= 0: + window_len += 1 + patchifier = self.diffusion_model.patchifier + latent_coords = patchifier.get_latent_coords(window_len, H, W, 1, cond_value.cond.device) + scale_factors = self.diffusion_model.vae_scale_factors + pixel_coords = comfy.ldm.lightricks.symmetric_patchifier.latent_to_pixel_coords( + latent_coords, + scale_factors, + causal_fix=self.diffusion_model.causal_temporal_positioning) + tokens = [] + for pos in kf_local_pos: + tokens.extend(range(pos * H * W, (pos + 1) * H * W)) + pixel_coords = pixel_coords[:, :, tokens, :] + + # Adjust spatial end positions for dilated (downscaled) guides. + # Each guide entry may have a different downscale factor; expand the + # per-entry factor to cover all tokens belonging to that entry. + downscale_factors = window.guide_downscale_factors + overlap_info = window.guide_overlap_info + if downscale_factors: + per_token_factor = [] + for (entry_idx, overlap_count), dsf in zip(overlap_info, downscale_factors): + per_token_factor.extend([dsf] * (overlap_count * H * W)) + factor_tensor = torch.tensor(per_token_factor, device=pixel_coords.device, dtype=pixel_coords.dtype) + spatial_end_offset = (factor_tensor.unsqueeze(0).unsqueeze(0).unsqueeze(-1) - 1) * torch.tensor( + scale_factors[1:], device=pixel_coords.device, dtype=pixel_coords.dtype, + ).view(1, -1, 1, 1) + pixel_coords[:, 1:, :, 1:] += spatial_end_offset + + B = cond_value.cond.shape[0] + if B > 1: + pixel_coords = pixel_coords.expand(B, -1, -1, -1) + return cond_value._copy_with(pixel_coords) + + # Guide attention entries — adjust per-guide counts based on window overlap + if cond_key == "guide_attention_entries": + overlap_info = window.guide_overlap_info + H, W = x_in.shape[3], x_in.shape[4] + new_entries = [] + for entry_idx, overlap_count in overlap_info: + e = cond_value.cond[entry_idx] + new_entries.append({**e, + "pre_filter_count": overlap_count * H * W, + "latent_shape": [overlap_count, H, W]}) + return cond_value._copy_with(new_entries) + + return None + class HunyuanVideo(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo) @@ -1517,8 +1655,26 @@ class WAN21(BaseModel): if reference_latents is not None: out['reference_latent'] = comfy.conds.CONDRegular(self.process_latent_in(reference_latents[-1])[:, :, 0]) + # In-context reference conditioning (Bernini) + context_latents = kwargs.get("context_latents", None) + if context_latents is not None: + out['context_latents'] = comfy.conds.CONDList([self.process_latent_in(l) for l in context_latents]) + return out + def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]): + # In-context cond slicing (Bernini) + if cond_key == "context_latents" and isinstance(getattr(cond_value, "cond", None), list): + dim = window.dim + out = [] + for lat in cond_value.cond: + if lat.ndim > dim and lat.shape[dim] > 1 and lat.shape[dim] == x_in.shape[dim]: + out.append(window.get_tensor(lat, device, dim=dim, retain_index_list=retain_index_list)) + else: + out.append(lat.to(device)) + return cond_value._copy_with(out) + return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list) + class WAN21_CausalAR(WAN21): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): @@ -1727,10 +1883,14 @@ class WAN21_SCAIL(WAN21): reference_latents = kwargs.get("reference_latents", None) if reference_latents is not None: - ref_latent = self.process_latent_in(reference_latents[-1]) - ref_mask = torch.ones_like(ref_latent[:, :4]) - ref_latent = torch.cat([ref_latent, ref_mask], dim=1) - out['reference_latent'] = comfy.conds.CONDRegular(ref_latent) + # SCAIL-2 multi-reference: reference_latents[0] is the primary ref, [1:] are additional + # references. Stack as [additional..., primary] so the primary stays adjacent to the video. + ordered = list(reference_latents[1:]) + list(reference_latents[:1]) + stacked = [] + for lat in ordered: + lat = self.process_latent_in(lat) + stacked.append(torch.cat([lat, torch.ones_like(lat[:, :4])], dim=1)) + out['reference_latent'] = comfy.conds.CONDRegular(torch.cat(stacked, dim=2)) pose_latents = kwargs.get("pose_video_latent", None) if pose_latents is not None: @@ -1753,6 +1913,99 @@ class WAN21_SCAIL(WAN21): return out +class WAN21_SCAIL2(WAN21_SCAIL): + """SCAIL-2: SCAIL-Preview + an additive binary multi-identity mask stream.""" + + def __init__(self, model_config, model_type=ModelType.FLOW, image_to_video=False, device=None): + super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.SCAIL2WanModel) + self.memory_usage_factor_conds = ("reference_latent", "pose_latents", "ref_mask_latents", "sam_latents") + self.memory_usage_shape_process = { + "pose_latents": lambda shape: [shape[0], shape[1], 1.5, shape[-2], shape[-1]], + "sam_latents": lambda shape: [shape[0], shape[1], 1.5, shape[-2], shape[-1]], + } + self.image_to_video = image_to_video + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + + driving_mask_28ch = kwargs.get("driving_mask_28ch", None) + if driving_mask_28ch is not None: + out['sam_latents'] = comfy.conds.CONDRegular(driving_mask_28ch.movedim(1, 2).contiguous()) + + # ref_mask_28ch holds one identity mask per stacked reference frame (additional refs first, then the primary ref), followed by zeros over the video frames. + ref_mask_28ch = kwargs.get("ref_mask_28ch", None) + if ref_mask_28ch is not None: + out['ref_mask_latents'] = comfy.conds.CONDRegular(ref_mask_28ch.movedim(1, 2).contiguous()) + + ref_mask_flag = kwargs.get("ref_mask_flag", None) + if ref_mask_flag is not None: + out['ref_mask_flag'] = comfy.conds.CONDConstant(ref_mask_flag) + + return out + + def extra_conds_shapes(self, **kwargs): + out = super().extra_conds_shapes(**kwargs) + driving_mask_28ch = kwargs.get("driving_mask_28ch", None) + if driving_mask_28ch is not None: + s = driving_mask_28ch.shape + out['sam_latents'] = [s[0], 28, s[1], s[3], s[4]] + ref_mask_28ch = kwargs.get("ref_mask_28ch", None) + if ref_mask_28ch is not None: + s = ref_mask_28ch.shape + out['ref_mask_latents'] = [s[0], 28, s[1], s[3], s[4]] + return out + + def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]): + if cond_key in ("sam_latents", "pose_latents"): + # Return sliced view omitting retain_index_list + return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=0) + if cond_key == "ref_mask_latents" and hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor): + # The ref mask is N leading ref frames padded with frames of zeros, so just grab the first frames for all windows + full_ref_mask = cond_value.cond + video_frame_count = x_in.shape[2] + ref_frame_count = full_ref_mask.shape[2] - video_frame_count + if ref_frame_count < 1: + return None + window_length = len(window.index_list) + + # Account for the causal anchor frame if it exists + anchor_index = getattr(window, "causal_anchor_index", None) + if anchor_index is not None and anchor_index >= 0: + window_length += 1 + + window_ref_mask = full_ref_mask[:, :, :window_length + ref_frame_count].to(device) + return cond_value._copy_with(window_ref_mask) + + return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list) + + def concat_cond(self, **kwargs): + # The 4 extra channels are the history_mask (1 at clean-anchor frames). + noise = kwargs.get("noise", None) + extra_channels = self.diffusion_model.patch_embedding.weight.shape[1] - noise.shape[1] + if extra_channels != 4: + return super().concat_cond(**kwargs) + + mask = kwargs.get("concat_mask", kwargs.get("denoise_mask", None)) + if mask is None: + return torch.zeros_like(noise)[:, :4] + + device = kwargs["device"] + if mask.shape[1] != 4: + mask = torch.mean(mask, dim=1, keepdim=True) + mask = 1.0 - mask + mask = utils.common_upscale(mask.to(device), noise.shape[-1], noise.shape[-2], "bilinear", "center") + if mask.shape[-3] < noise.shape[-3]: + mask = torch.nn.functional.pad(mask, (0, 0, 0, 0, 0, noise.shape[-3] - mask.shape[-3]), mode='constant', value=0) + if mask.shape[1] == 1: + mask = mask.repeat(1, 4, 1, 1, 1) + mask = utils.resize_to_batch_size(mask, noise.shape[0]) + return mask + + def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs): + # Hold anchor constant across all sigmas instead of base sigma*noise + (1-sigma)*latent_image. + return latent_image + + class WAN22_WanDancer(WAN21): def __init__(self, model_config, model_type=ModelType.FLOW, image_to_video=True, device=None): super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model_wandancer.WanDancerModel) @@ -1986,6 +2239,11 @@ class Omnigen2(BaseModel): out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16]) return out +class Boogu(Omnigen2): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super(Omnigen2, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.boogu.model.BooguTransformer2DModel) + self.memory_usage_factor_conds = ("ref_latents",) + class QwenImage(BaseModel): def __init__(self, model_config, model_type=ModelType.FLUX, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.qwen_image.model.QwenImageTransformer2DModel) @@ -2018,6 +2276,32 @@ class QwenImage(BaseModel): out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16]) return out +class Ideogram4(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.ideogram4.model.Ideogram4Transformer2DModel) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + attention_mask = kwargs.get("attention_mask", None) + if attention_mask is not None: + if torch.numel(attention_mask) != attention_mask.sum(): + out['attention_mask'] = comfy.conds.CONDRegular(attention_mask) + cross_attn = kwargs.get("cross_attn", None) + if cross_attn is not None: + out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) + return out + +class Krea2(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLUX, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.krea2.model.SingleStreamDiT) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + cross_attn = kwargs.get("cross_attn", None) + if cross_attn is not None: + out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) + return out + class HunyuanImage21(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo) @@ -2211,6 +2495,12 @@ class RT_DETR_v4(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.rt_detr.rtdetr_v4.RTv4) + +class DepthAnything3(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super().__init__(model_config, model_type, device=device, + unet_model=comfy.ldm.depth_anything_3.model.DepthAnything3Net) + class ErnieImage(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.ernie.model.ErnieImageModel) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 24e742a7f..70c8625e3 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -470,15 +470,46 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): # PiD (Pixel Diffusion Decoder). Must check BEFORE plain PixelDiT_T2I. _lq_w_key = '{}lq_proj.latent_proj.0.weight'.format(key_prefix) if _lq_w_key in state_dict_keys: - in_ch = int(state_dict[_lq_w_key].shape[1]) + latent_proj_in_channels = int(state_dict[_lq_w_key].shape[1]) + hidden_dim = int(state_dict[_lq_w_key].shape[0]) _gate_prefix = '{}lq_proj.gate_modules.'.format(key_prefix) num_gates = len({k[len(_gate_prefix):].split('.')[0] for k in state_dict_keys if k.startswith(_gate_prefix)}) + pid_v1_5 = '{}lq_proj.pit_head.weight'.format(key_prefix) in state_dict_keys dit_config = {"image_model": "pid", - "lq_latent_channels": in_ch, - "latent_spatial_down_factor": 16 if in_ch >= 64 else 8} + "lq_hidden_dim": hidden_dim} if num_gates > 0: dit_config["lq_interval"] = (14 + num_gates - 1) // num_gates + if pid_v1_5: + pid_v1_5_variants = { + 16: { # Flux and QwenImage + "lq_latent_channels": 16, + "latent_spatial_down_factor": 8, + "lq_latent_unpatchify_factor": 1, + }, + 32: { # Flux2 after 2x latent unpatchify + "lq_latent_channels": 128, + "latent_spatial_down_factor": 16, + "lq_latent_unpatchify_factor": 2, + }, + } + variant = pid_v1_5_variants.get(latent_proj_in_channels) + if variant is None: + raise ValueError(f"Unsupported PiD v1.5 latent projection with {latent_proj_in_channels} input channels") + gate_weight = state_dict['{}lq_proj.gate_modules.0.content_proj.weight'.format(key_prefix)] + dit_config.update(variant) + dit_config.update({ + "lq_conv_padding_mode": "replicate", + "lq_gate_per_token": gate_weight.shape[0] == 1, + "pit_lq_inject": True, + "rope_ref_h": 2048, + "rope_ref_w": 2048, + }) + else: + dit_config.update({ + "lq_latent_channels": latent_proj_in_channels, + "latent_spatial_down_factor": 16 if latent_proj_in_channels >= 64 else 8, + }) return dit_config if '{}core.pixel_embedder.proj.weight'.format(key_prefix) in state_dict_keys: # PixelDiT T2I @@ -598,6 +629,44 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): return dit_config + seedvr2_7b_separate_key = "{}blocks.35.mlp.vid.proj_out.weight".format(key_prefix) + if seedvr2_7b_separate_key in state_dict_keys and state_dict[seedvr2_7b_separate_key].shape[0] == 3072: # seedvr2 7b + dit_config = {} + dit_config["image_model"] = "seedvr2" + dit_config["vid_dim"] = 3072 + dit_config["heads"] = 24 + dit_config["num_layers"] = 36 + # This checkpoint uses separate vid/txt MMModule keys in every block. + dit_config["mm_layers"] = 36 + dit_config["norm_eps"] = 1e-5 + dit_config["rope_type"] = "rope3d" + dit_config["rope_dim"] = 64 + dit_config["mlp_type"] = "normal" + return dit_config + if "{}blocks.35.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 7b + dit_config = {} + dit_config["image_model"] = "seedvr2" + dit_config["vid_dim"] = 3072 + dit_config["heads"] = 24 + dit_config["num_layers"] = 36 + # This checkpoint uses shared all.* MMModule keys after the initial blocks. + dit_config["mm_layers"] = 10 + dit_config["norm_eps"] = 1e-5 + dit_config["rope_type"] = "rope3d" + dit_config["rope_dim"] = 64 + dit_config["mlp_type"] = "swiglu" + return dit_config + if "{}blocks.31.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 3b + dit_config = {} + dit_config["image_model"] = "seedvr2" + dit_config["vid_dim"] = 2560 + dit_config["heads"] = 20 + dit_config["num_layers"] = 32 + dit_config["norm_eps"] = 1.0e-05 + dit_config["mlp_type"] = "swiglu" + dit_config["vid_out_norm"] = True + return dit_config + if '{}head.modulation'.format(key_prefix) in state_dict_keys: # Wan 2.1 dit_config = {} dit_config["image_model"] = "wan2.1" @@ -630,6 +699,8 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["model_type"] = "humo" elif '{}face_adapter.fuser_blocks.0.k_norm.weight'.format(key_prefix) in state_dict_keys: dit_config["model_type"] = "animate" + elif '{}patch_embedding_mask.weight'.format(key_prefix) in state_dict_keys: + dit_config["model_type"] = "scail2" elif '{}patch_embedding_pose.weight'.format(key_prefix) in state_dict_keys: dit_config["model_type"] = "scail" elif '{}patch_embedding_global.weight'.format(key_prefix) in state_dict_keys: @@ -759,6 +830,16 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): return dit_config + if '{}double_stream_layers.0.img_instruct_attn.processor.img_to_q.weight'.format(key_prefix) in state_dict_keys: # Boogu-Image (OmniGen2 derivative + dual-stream stage) + dit_config = {} + dit_config["image_model"] = "boogu" + dit_config["hidden_size"] = state_dict['{}x_embedder.weight'.format(key_prefix)].shape[0] + dit_config["num_layers"] = count_blocks(state_dict_keys, '{}single_stream_layers.'.format(key_prefix) + '{}.') + dit_config["num_double_stream_layers"] = count_blocks(state_dict_keys, '{}double_stream_layers.'.format(key_prefix) + '{}.') + dit_config["num_refiner_layers"] = count_blocks(state_dict_keys, '{}noise_refiner.'.format(key_prefix) + '{}.') + dit_config["instruction_feat_dim"] = state_dict['{}time_caption_embed.caption_embedder.0.weight'.format(key_prefix)].shape[0] + return dit_config + if '{}time_caption_embed.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: # Omnigen2 dit_config = {} dit_config["image_model"] = "omnigen2" @@ -815,6 +896,28 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["default_ref_method"] = "negative_index" return dit_config + if '{}embed_image_indicator.weight'.format(key_prefix) in state_dict_keys: # Ideogram 4 + dit_config = {} + dit_config["image_model"] = "ideogram4" + dit_config["in_channels"] = state_dict['{}input_proj.weight'.format(key_prefix)].shape[1] + dit_config["num_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.') + return dit_config + + if '{}txtfusion.projector.weight'.format(key_prefix) in state_dict_keys: # Krea 2 (K2) + dit_config = {} + dit_config["image_model"] = "krea2" + head_dim = 128 + first_w = state_dict['{}first.weight'.format(key_prefix)] # (features, channels*patch^2) + dit_config["features"] = first_w.shape[0] + dit_config["channels"] = first_w.shape[1] // (2 * 2) # patch=2 + dit_config["patch"] = 2 + dit_config["layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.') + dit_config["heads"] = state_dict['{}blocks.0.attn.wq.weight'.format(key_prefix)].shape[0] // head_dim + dit_config["kvheads"] = state_dict['{}blocks.0.attn.wk.weight'.format(key_prefix)].shape[0] // head_dim + dit_config["txtlayers"] = state_dict['{}txtfusion.projector.weight'.format(key_prefix)].shape[1] + dit_config["txtdim"] = state_dict['{}txtfusion.layerwise_blocks.0.prenorm.scale'.format(key_prefix)].shape[0] + return dit_config + if '{}visual_transformer_blocks.0.cross_attention.key_norm.weight'.format(key_prefix) in state_dict_keys: # Kandinsky 5 dit_config = {} model_dim = state_dict['{}visual_embeddings.in_layer.bias'.format(key_prefix)].shape[0] @@ -853,6 +956,95 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["enc_h"] = state_dict['{}encoder.pan_blocks.1.cv4.conv.weight'.format(key_prefix)].shape[0] return dit_config + # Depth Anything 3 (repackaged to ComfyUI's native Dinov2Model layout via scripts/convert_da3.py) + if '{}backbone.embeddings.patch_embeddings.projection.weight'.format(key_prefix) in state_dict_keys: + dit_config = {} + dit_config["image_model"] = "DepthAnything3" + + patch_w = state_dict['{}backbone.embeddings.patch_embeddings.projection.weight'.format(key_prefix)] + embed_dim = patch_w.shape[0] + depth = count_blocks(state_dict_keys, '{}backbone.encoder.layer.'.format(key_prefix) + '{}.') + + # Backbone preset is determined by embed_dim (matches vits/vitb/vitl/vitg). + backbone_name = {384: "vits", 768: "vitb", 1024: "vitl", 1536: "vitg"}.get(embed_dim) + if backbone_name is None: + return None + dit_config["backbone_name"] = backbone_name + + # Detect DA3 extensions on top of vanilla DINOv2. + has_camera_token = '{}backbone.embeddings.camera_token'.format(key_prefix) in state_dict_keys + # qk-norm shows up as `attention.q_norm.weight` on enabled blocks. + qknorm_indices = [ + i for i in range(depth) + if '{}backbone.encoder.layer.{}.attention.q_norm.weight'.format(key_prefix, i) in state_dict_keys + ] + qknorm_start = qknorm_indices[0] if qknorm_indices else -1 + + # The DA3 main-series configs always set alt_start == qknorm_start == rope_start. + # cat_token=True is implied by the presence of camera_token. + if has_camera_token: + dit_config["alt_start"] = qknorm_start + dit_config["rope_start"] = qknorm_start + dit_config["qknorm_start"] = qknorm_start + dit_config["cat_token"] = True + else: + dit_config["alt_start"] = -1 + dit_config["rope_start"] = -1 + dit_config["qknorm_start"] = -1 + dit_config["cat_token"] = False + + # Detect head type and config. + has_aux = '{}head.scratch.refinenet1_aux.out_conv.weight'.format(key_prefix) in state_dict_keys + dit_config["head_dim_in"] = state_dict['{}head.projects.0.weight'.format(key_prefix)].shape[1] + dit_config["head_features"] = state_dict['{}head.scratch.refinenet1.out_conv.weight'.format(key_prefix)].shape[0] + dit_config["head_out_channels"] = [ + state_dict['{}head.projects.{}.weight'.format(key_prefix, i)].shape[0] + for i in range(4) + ] + if has_aux: + # DualDPT: dim_in = 2 * embed_dim (because cat_token doubles token width). + dit_config["head_type"] = "dualdpt" + dit_config["head_output_dim"] = 2 + dit_config["head_use_sky_head"] = False + else: + dit_config["head_type"] = "dpt" + dit_config["head_output_dim"] = state_dict[ + '{}head.scratch.output_conv2.2.weight'.format(key_prefix) + ].shape[0] + dit_config["head_use_sky_head"] = ( + '{}head.scratch.sky_output_conv2.0.weight'.format(key_prefix) in state_dict_keys + ) + + # out_layers: hard-coded per upstream YAML config (depth-aware default). + if depth >= 24: + # vitl: depths used vary between DA3-Large (DualDPT) and Mono/Metric (DPT). + if has_aux: + dit_config["out_layers"] = [11, 15, 19, 23] + else: + dit_config["out_layers"] = [4, 11, 17, 23] + else: + # vits/vitb: 12 blocks + dit_config["out_layers"] = [5, 7, 9, 11] + + # Camera encoder/decoder presence (multi-view + pose path). + has_cam_enc = '{}cam_enc.token_norm.weight'.format(key_prefix) in state_dict_keys + has_cam_dec = '{}cam_dec.fc_t.weight'.format(key_prefix) in state_dict_keys + dit_config["has_cam_enc"] = has_cam_enc + dit_config["has_cam_dec"] = has_cam_dec + if has_cam_enc: + cam_enc_w = state_dict.get( + '{}cam_enc.pose_branch.fc2.weight'.format(key_prefix) + ) + if cam_enc_w is not None: + dit_config["cam_dim_out"] = cam_enc_w.shape[0] + if has_cam_dec: + cam_dec_w = state_dict.get( + '{}cam_dec.fc_t.weight'.format(key_prefix) + ) + if cam_dec_w is not None: + dit_config["cam_dec_dim_in"] = cam_dec_w.shape[1] + return dit_config + if '{}layers.0.mlp.linear_fc2.weight'.format(key_prefix) in state_dict_keys: # Ernie Image dit_config = {} dit_config["image_model"] = "ernie" @@ -996,9 +1188,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): return unet_config -def model_config_from_unet_config(unet_config, state_dict=None): + +def model_config_from_unet_config(unet_config, state_dict=None, unet_key_prefix=""): for model_config in comfy.supported_models.models: - if model_config.matches(unet_config, state_dict): + if model_config.matches(unet_config, state_dict, unet_key_prefix=unet_key_prefix): return model_config(unet_config) logging.error("no match {}".format(unet_config)) @@ -1008,7 +1201,7 @@ def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=Fal unet_config = detect_unet_config(state_dict, unet_key_prefix, metadata=metadata) if unet_config is None: return None - model_config = model_config_from_unet_config(unet_config, state_dict) + model_config = model_config_from_unet_config(unet_config, state_dict, unet_key_prefix) if model_config is None and use_base_if_no_match: model_config = comfy.supported_models_base.BASE(unet_config) diff --git a/comfy/model_management.py b/comfy/model_management.py index dfd58bf1b..222005b6f 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -534,8 +534,10 @@ try: except: pass -if torch.cuda.is_available() and torch.backends.cudnn.is_available() and PerformanceFeature.AutoTune in args.fast: - torch.backends.cudnn.benchmark = True + +def set_cudnn_benchmark(): + if torch.cuda.is_available() and torch.backends.cudnn.is_available(): + torch.backends.cudnn.benchmark = PerformanceFeature.AutoTune in args.fast try: if torch_version_numeric >= (2, 5): @@ -614,6 +616,8 @@ PIN_PRESSURE_HYSTERESIS = 256 * 1024 * 1024 #Freeing registerables on pressure does imply a GPU sync, so go big on #the hysteresis so each expensive sync gives us back a good chunk. REGISTERABLE_PIN_HYSTERESIS = 2048 * 1024 * 1024 +WINDOWS_PIN_EVICTION_SWAP_PERCENT = 5.0 +WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE = 512 * 1024 ** 2 def module_size(module): module_mem = 0 @@ -640,7 +644,18 @@ def free_pins(size, evict_active=False): size -= freed return freed_total +def should_free_pins_for_ram_pressure(shortfall): + if shortfall <= 0: + return False + if not WINDOWS: + return True + if psutil.virtual_memory().available < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE: + return True + return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT + def ensure_pin_budget(size, evict_active=False): + if args.high_ram: + return True if args.fast_disk: shortfall = TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY else: @@ -651,8 +666,7 @@ def ensure_pin_budget(size, evict_active=False): to_free = shortfall + PIN_PRESSURE_HYSTERESIS return free_pins(to_free, evict_active=evict_active) >= shortfall -def ensure_pin_registerable(size, evict_active=True): - shortfall = TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY +def free_registrations(shortfall, evict_active=True): if MAX_PINNED_MEMORY <= 0: return False if shortfall <= 0: @@ -674,6 +688,9 @@ def ensure_pin_registerable(size, evict_active=True): return True return shortfall <= REGISTERABLE_PIN_HYSTERESIS +def ensure_pin_registerable(size, evict_active=True): + return free_registrations(TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY, evict_active=evict_active) + class LoadedModel: def __init__(self, model: ModelPatcher): self._set_model(model) @@ -956,8 +973,6 @@ def loaded_models(only_currently_used=False): def cleanup_models_gc(): do_gc = False - reset_cast_buffers() - for i in range(len(current_loaded_models)): cur = current_loaded_models[i] if cur.is_dead(): @@ -1494,6 +1509,8 @@ if not args.disable_pinned_memory: PINNING_ALLOWED_TYPES = set(["Tensor", "Parameter", "QuantizedTensor"]) def pinned_hostbuf_size(size): + if args.high_ram: + return max(0, int(size * 2)) return max(0, int(min(size, MAX_PINNED_MEMORY) * 2)) def discard_cuda_async_error(): diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index b716a69e2..d70b42bf8 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -379,10 +379,11 @@ class ModelPatcher: def get_clone_model_override(self): return self.model, (self.backup, self.backup_buffers, self.object_patches_backup, self.pinned) - def clone(self, disable_dynamic=False, model_override=None): + def clone(self, disable_dynamic=False, model_override=None, force_deepcopy=False): class_ = self.__class__ - if self.is_dynamic() and disable_dynamic: - class_ = ModelPatcher + if self.is_dynamic() and disable_dynamic or force_deepcopy: + if self.is_dynamic() and disable_dynamic: + class_ = ModelPatcher if model_override is None: if self.cached_patcher_init is None: raise RuntimeError("Cannot create non-dynamic delegate: cached_patcher_init is not initialized.") diff --git a/comfy/multigpu.py b/comfy/multigpu.py index bb9d334d3..2b6d8260d 100644 --- a/comfy/multigpu.py +++ b/comfy/multigpu.py @@ -54,6 +54,8 @@ class MultiGPUThreadPool: try: result = fn(*args, **kwargs) result_q.put((result, None)) + except comfy.model_management.InterruptProcessingException as e: + result_q.put((None, e)) except Exception as e: result_q.put((None, e)) diff --git a/comfy/ops.py b/comfy/ops.py index 119177c37..13c2604fb 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -174,13 +174,15 @@ def cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blockin elif xfer_dest2 is not None: xfer_source.prepare(xfer_dest2, stream, copy=True, commit=False) return + else: + return comfy.model_management.cast_to_gathered(xfer_source, xfer_dest, non_blocking=non_blocking, stream=stream, r2=xfer_dest2) def handle_pin(m, pin, source, dest, subset="weights", size=None): if pin is not None: cast_maybe_lowvram_patch([pin], dest, offload_stream) return - if signature is None: + if signature is None or args.high_ram: comfy.pinned_memory.pin_memory(m, subset=subset, size=size) pin = comfy.pinned_memory.get_pin(m, subset=subset) cast_maybe_lowvram_patch(source, pin, offload_stream, xfer_dest2=dest) @@ -256,7 +258,7 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, w if (want_requant and len(fns) == 0 or update_weight): seed = comfy.utils.string_to_seed(s.seed_key) if isinstance(orig, QuantizedTensor): - y = QuantizedTensor.from_float(x, s.layout_type, scale="recalculate", stochastic_rounding=seed) + y = orig.requantize_from_float(x, scale="recalculate", stochastic_rounding=seed) else: y = comfy.float.stochastic_rounding(x, orig.dtype, seed=seed) if want_requant and len(fns) == 0: @@ -299,21 +301,21 @@ def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None, of non_blocking = comfy.model_management.device_supports_non_blocking(device) - if hasattr(s, "_v"): + if hasattr(s, "_v") and comfy.model_management.is_device_cpu(device): #vbar doesn't support CPU weights, but some custom nodes have weird paths #that might switch the layer to the CPU and expect it to work. We have to take #a clone conservatively as we are mmapped and some SFT files are packed misaligned #If you are a custom node author reading this, please move your layer to the GPU #or declare your ModelPatcher as CPU in the first place. - if comfy.model_management.is_device_cpu(device): - materialize_meta_param(s, ["weight", "bias"]) - weight = s.weight.to(dtype=dtype, copy=True) - if isinstance(weight, QuantizedTensor): - weight = weight.dequantize() - bias = s.bias.to(dtype=bias_dtype, copy=True) if s.bias is not None else None - return format_return((weight, bias, (None, None, None)), offloadable) + materialize_meta_param(s, ["weight", "bias"]) + weight = s.weight.to(dtype=dtype, copy=True) + if isinstance(weight, QuantizedTensor): + weight = weight.dequantize() + bias = s.bias.to(dtype=bias_dtype, copy=True) if s.bias is not None else None + return format_return((weight, bias, (None, None, None)), offloadable) + elif hasattr(s, "_v") and s.weight.device != device: prefetched = hasattr(s, "_prefetch") offload_stream = None offload_device = None @@ -1089,6 +1091,34 @@ def _load_quantized_module(module, super_load, state_dict, prefix, local_metadat if ts is None or bs is None: raise ValueError(f"Missing NVFP4 scales for layer {layer_name}") scales = {"scale": ts, "block_scale": bs} + elif module.quant_format == "int8_tensorwise": + scale = pop_scale("weight_scale") + if scale is None: + raise ValueError(f"Missing INT8 weight scale for layer {layer_name}") + scales = {"scale": scale} + params_conf = layer_conf.get("params", {}) + if not isinstance(params_conf, dict): + params_conf = {} + if layer_conf.get("convrot", params_conf.get("convrot", False)): + scales["convrot"] = True + scales["convrot_groupsize"] = int( + layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256)) + ) + elif module.quant_format == "convrot_w4a4": + scale = pop_scale("weight_scale") + if scale is None: + raise ValueError(f"Missing ConvRot W4A4 weight scale for layer {layer_name}") + params_conf = layer_conf.get("params", {}) + if not isinstance(params_conf, dict): + params_conf = {} + scales = { + "scale": scale, + "convrot_groupsize": int( + layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256)) + ), + "quant_group_size": 64, + "linear_dtype": layer_conf.get("linear_dtype", params_conf.get("linear_dtype", "int4")), + } else: raise ValueError(f"Unsupported quantization format: {module.quant_format}") @@ -1131,6 +1161,15 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extr quant_conf = {"format": module.quant_format} if getattr(module, '_full_precision_mm_config', False): quant_conf["full_precision_matrix_mult"] = True + params = getattr(module.weight, "_params", None) + if module.quant_format == "int8_tensorwise" and getattr(params, "convrot", False): + quant_conf["convrot"] = True + quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256) + elif module.quant_format == "convrot_w4a4": + quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256) + linear_dtype = getattr(params, "linear_dtype", "int4") + if linear_dtype != "int4": + quant_conf["linear_dtype"] = linear_dtype if extra_quant_conf: quant_conf.update(extra_quant_conf) sd[f"{prefix}comfy_quant"] = torch.tensor(list(json.dumps(quant_conf).encode("utf-8")), dtype=torch.uint8) @@ -1183,8 +1222,33 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def _forward(self, input, weight, bias): return torch.nn.functional.linear(input, weight, bias) - def forward_comfy_cast_weights(self, input, compute_dtype=None, want_requant=False): - weight, bias, offload_stream = cast_bias_weight(self, input, offloadable=True, compute_dtype=compute_dtype, want_requant=want_requant) + def forward_comfy_cast_weights( + self, + input, + compute_dtype=None, + want_requant=False, + weight_only_quant=False, + ): + if weight_only_quant: + weight, bias, offload_stream = cast_bias_weight( + self, + input=None, + dtype=self.weight.dtype, + device=input.device, + bias_dtype=input.dtype, + offloadable=True, + compute_dtype=compute_dtype, + want_requant=True, + ) + weight = weight.to(dtype=input.dtype) + else: + weight, bias, offload_stream = cast_bias_weight( + self, + input, + offloadable=True, + compute_dtype=compute_dtype, + want_requant=want_requant, + ) x = self._forward(input, weight, bias) uncast_bias_weight(self, weight, bias, offload_stream) return x @@ -1193,7 +1257,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec run_every_op() input_shape = input.shape - reshaped_3d = False + reshaped_nd = False #If cast needs to apply lora, it should be done in the compute dtype compute_dtype = input.dtype @@ -1203,9 +1267,10 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec not getattr(self, 'comfy_force_cast_weights', False) and len(self.weight_function) == 0 and len(self.bias_function) == 0 ) + quantize_input = QUANT_ALGOS.get(getattr(self, 'quant_format', None), {}).get("quantize_input", True) # Training path: quantized forward with compute_dtype backward via autograd function - if (input.requires_grad and _use_quantized): + if (input.requires_grad and _use_quantized and quantize_input): weight, bias, offload_stream = cast_bias_weight( self, @@ -1227,25 +1292,31 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec return output # Inference path (unchanged) - if _use_quantized: + if _use_quantized and quantize_input: - # Reshape 3D tensors to 2D for quantization (needed for NVFP4 and others) - input_reshaped = input.reshape(-1, input_shape[2]) if input.ndim == 3 else input + # Reshape >=3D tensors to 2D for quantization (needed for NVFP4 and others) + input_reshaped = input.reshape(-1, input_shape[-1]) if input.ndim >= 3 else input # Fall back to non-quantized for non-2D tensors if input_reshaped.ndim == 2: - reshaped_3d = input.ndim == 3 + reshaped_nd = input.ndim >= 3 # dtype is now implicit in the layout class scale = getattr(self, 'input_scale', None) if scale is not None: scale = comfy.model_management.cast_to_device(scale, input.device, None) input = QuantizedTensor.from_float(input_reshaped, self.layout_type, scale=scale) - output = self.forward_comfy_cast_weights(input, compute_dtype, want_requant=isinstance(input, QuantizedTensor)) + weight_only_quant = _use_quantized and not quantize_input and isinstance(self.weight, QuantizedTensor) + output = self.forward_comfy_cast_weights( + input, + compute_dtype, + want_requant=isinstance(input, QuantizedTensor), + weight_only_quant=weight_only_quant, + ) - # Reshape output back to 3D if input was 3D - if reshaped_3d: - output = output.reshape((input_shape[0], input_shape[1], self.weight.shape[0])) + # Reshape output back to original rank if input was >2D + if reshaped_nd: + output = output.reshape((*input_shape[:-1], self.weight.shape[0])) return output @@ -1257,8 +1328,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def set_weight(self, weight, inplace_update=False, seed=None, return_weight=False, **kwargs): if getattr(self, 'layout_type', None) is not None: - # dtype is now implicit in the layout class - weight = QuantizedTensor.from_float(weight, self.layout_type, scale="recalculate", stochastic_rounding=seed, inplace_ops=True).to(self.weight.dtype) + weight = self.weight.requantize_from_float(weight, scale="recalculate", stochastic_rounding=seed, inplace_ops=True).to(self.weight.dtype) else: weight = weight.to(self.weight.dtype) if return_weight: @@ -1380,6 +1450,12 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec } if hasattr(params, "block_scale"): # NVFP4 kwargs["block_scale"] = params.block_scale[i] + if hasattr(params, "quant_group_size"): + kwargs["quant_group_size"] = params.quant_group_size + if hasattr(params, "convrot_groupsize"): + kwargs["convrot_groupsize"] = params.convrot_groupsize + if hasattr(params, "linear_dtype"): + kwargs["linear_dtype"] = params.linear_dtype return QuantizedTensor(weight._qdata[i], weight._layout_cls, type(params)(**kwargs)) def state_dict(self, *args, destination=None, prefix="", **kwargs): diff --git a/comfy/pinned_memory.py b/comfy/pinned_memory.py index ffe12e0dc..cb77c517a 100644 --- a/comfy/pinned_memory.py +++ b/comfy/pinned_memory.py @@ -89,13 +89,26 @@ def pin_memory(module, subset="weights", size=None): not comfy.model_management.ensure_pin_registerable(registerable_size)): return _steal_pin(module, stack, buckets, size, priority) + extended = False try: - hostbuf.extend(size=size) + hostbuf.extend(size=size, register=False) + extended = True + pin = comfy_aimdo.torch.hostbuf_to_tensor(hostbuf)[offset:offset + size] + pin.untyped_storage()._comfy_hostbuf = hostbuf + if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: + comfy.model_management.discard_cuda_async_error() + comfy.model_management.free_registrations(size) + if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: + comfy.model_management.discard_cuda_async_error() + del pin + hostbuf.truncate(offset, do_unregister=False) + return _steal_pin(module, stack, buckets, size, priority) except RuntimeError: + if extended: + hostbuf.truncate(offset, do_unregister=False) return _steal_pin(module, stack, buckets, size, priority) - module._pin = comfy_aimdo.torch.hostbuf_to_tensor(hostbuf)[offset:offset + size] - module._pin.untyped_storage()._comfy_hostbuf = hostbuf + module._pin = pin stack.append((module, offset)) module._pin_registered = True module._pin_stack_index = len(stack) - 1 diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py index b90bcfd25..15f9b1fdb 100644 --- a/comfy/quant_ops.py +++ b/comfy/quant_ops.py @@ -3,6 +3,22 @@ import logging from comfy.cli_args import args + +def _rocm_kitchen_arch_supported(): + """comfy-kitchen's INT8 Triton kernels compile tl.dot to matrix-core instructions. + RDNA3/3.5/4 (gfx11xx/gfx12xx) have WMMA and CDNA (gfx9xx) has MFMA; RDNA1/RDNA2 + (gfx10xx) have neither, so the INT8 path hangs the GPU there. Gates the automatic + ROCm default so those cards stay on the eager fallback (an explicit + --enable-triton-backend still forces it on any arch).""" + try: + arch = torch.cuda.get_device_properties(torch.cuda.current_device()).gcnArchName.split(":")[0] + except Exception: + return False + if arch.startswith(("gfx11", "gfx12")): + return True + return arch in ("gfx908", "gfx90a", "gfx940", "gfx941", "gfx942", "gfx950") + + try: import comfy_kitchen as ck from comfy_kitchen.tensor import ( @@ -10,6 +26,8 @@ try: QuantizedLayout, TensorCoreFP8Layout as _CKFp8Layout, TensorCoreNVFP4Layout as _CKNvfp4Layout, + TensorCoreConvRotW4A4Layout as _CKTensorCoreConvRotW4A4Layout, + TensorWiseINT8Layout as _CKTensorWiseINT8Layout, register_layout_op, register_layout_class, get_layout_class, @@ -23,10 +41,22 @@ try: ck.registry.disable("cuda") logging.warning("WARNING: You need pytorch with cu130 or higher to use optimized CUDA operations.") - if args.enable_triton_backend: + # On ROCm/AMD the CUDA backend is unavailable, so Triton is the only accelerated + # comfy-kitchen backend. Enable it by default there, but only on Triton >= 3.7 AND a + # matrix-core GPU (RDNA3+ WMMA gfx11xx/gfx12xx, CDNA MFMA gfx9xx). RDNA1/RDNA2 + # (gfx10xx) have no WMMA -> the INT8 tl.dot path hangs the GPU, so they stay eager. + # older Triton lacks libdevice.rint on the HIP backend and hard-crashes the INT8 path. + if args.disable_triton_backend: + ck.registry.disable("triton") + elif args.enable_triton_backend: # or (torch.version.hip is not None and _rocm_kitchen_arch_supported()): try: import triton - logging.info("Found triton %s. Enabling comfy-kitchen triton backend.", triton.__version__) + triton_version = tuple(int(v) for v in triton.__version__.split(".")[:2]) + if args.enable_triton_backend or triton_version >= (3, 7): + logging.info("Found triton %s. Enabling comfy-kitchen triton backend.", triton.__version__) + else: + logging.info("Triton %s is too old for the ROCm INT8 path (needs >= 3.7); comfy-kitchen triton backend disabled.", triton.__version__) + ck.registry.disable("triton") except ImportError as e: logging.error(f"Failed to import triton, Error: {e}, the comfy-kitchen triton backend will not be available.") ck.registry.disable("triton") @@ -47,6 +77,12 @@ except ImportError as e: class _CKNvfp4Layout: pass + class _CKTensorWiseINT8Layout: + pass + + class _CKTensorCoreConvRotW4A4Layout: + pass + def register_layout_class(name, cls): pass @@ -174,6 +210,8 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase): # Backward compatibility alias - default to E4M3 TensorCoreFP8Layout = TensorCoreFP8E4M3Layout +TensorWiseINT8Layout = _CKTensorWiseINT8Layout +TensorCoreConvRotW4A4Layout = _CKTensorCoreConvRotW4A4Layout # ============================================================================== @@ -184,6 +222,8 @@ register_layout_class("TensorCoreFP8Layout", TensorCoreFP8Layout) register_layout_class("TensorCoreFP8E4M3Layout", TensorCoreFP8E4M3Layout) register_layout_class("TensorCoreFP8E5M2Layout", TensorCoreFP8E5M2Layout) register_layout_class("TensorCoreNVFP4Layout", TensorCoreNVFP4Layout) +register_layout_class("TensorWiseINT8Layout", _CKTensorWiseINT8Layout) +register_layout_class("TensorCoreConvRotW4A4Layout", _CKTensorCoreConvRotW4A4Layout) if _CK_MXFP8_AVAILABLE: register_layout_class("TensorCoreMXFP8Layout", TensorCoreMXFP8Layout) @@ -214,6 +254,20 @@ if _CK_MXFP8_AVAILABLE: "group_size": 32, } +QUANT_ALGOS["int8_tensorwise"] = { + "storage_t": torch.int8, + "parameters": {"weight_scale"}, + "comfy_tensor_layout": "TensorWiseINT8Layout", + "quantize_input": False, +} + +QUANT_ALGOS["convrot_w4a4"] = { + "storage_t": torch.int8, + "parameters": {"weight_scale"}, + "comfy_tensor_layout": "TensorCoreConvRotW4A4Layout", + "quantize_input": False, +} + # ============================================================================== # Re-exports for backward compatibility @@ -226,6 +280,8 @@ __all__ = [ "TensorCoreFP8E4M3Layout", "TensorCoreFP8E5M2Layout", "TensorCoreNVFP4Layout", + "TensorCoreConvRotW4A4Layout", + "TensorWiseINT8Layout", "QUANT_ALGOS", "register_layout_op", ] diff --git a/comfy/sd.py b/comfy/sd.py index 9a2d31930..4a0742e7a 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -16,6 +16,7 @@ import comfy.ldm.cosmos.vae import comfy.ldm.wan.vae import comfy.ldm.wan.vae2_2 import comfy.ldm.hunyuan3d.vae +import comfy.ldm.seedvr.vae import comfy.ldm.triposplat.vae import comfy.ldm.ace.vae.music_dcae_pipeline import comfy.ldm.cogvideo.vae @@ -58,6 +59,8 @@ import comfy.text_encoders.omnigen2 import comfy.text_encoders.qwen_image import comfy.text_encoders.hunyuan_image import comfy.text_encoders.z_image +import comfy.text_encoders.krea2 +import comfy.text_encoders.ideogram4 import comfy.text_encoders.ovis import comfy.text_encoders.kandinsky5 import comfy.text_encoders.jina_clip_2 @@ -66,6 +69,8 @@ import comfy.text_encoders.anima import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image import comfy.text_encoders.qwen35 +import comfy.text_encoders.qwen3vl +import comfy.text_encoders.boogu import comfy.text_encoders.ernie import comfy.text_encoders.gemma4 import comfy.text_encoders.cogvideo @@ -464,9 +469,13 @@ class CLIP: def decode(self, token_ids, skip_special_tokens=True): return self.tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens) + def is_dynamic(self): + return self.patcher.is_dynamic() + class VAE: def __init__(self, sd=None, device=None, config=None, dtype=None, metadata=None): - if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format + is_seedvr2_vae = "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd + if not is_seedvr2_vae and 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format sd = diffusers_convert.convert_vae_state_dict(sd) if model_management.is_amd(): @@ -493,6 +502,8 @@ class VAE: self.upscale_index_formula = None self.extra_1d_channel = None self.crop_input = True + self.handles_tiling = False + self.format_encoded = None self.audio_sample_rate = 44100 @@ -539,6 +550,22 @@ class VAE: self.first_stage_model = StageC_coder() self.downscale_ratio = 32 self.latent_channels = 16 + elif "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd: # seedvr2 + self.first_stage_model = comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper() + self.latent_channels = comfy.ldm.seedvr.vae.SEEDVR2_LATENT_CHANNELS + self.latent_dim = 3 + self.disable_offload = True + self.memory_used_decode = lambda shape, dtype: self.first_stage_model.comfy_memory_used_decode(shape) + self.memory_used_encode = lambda shape, dtype: (max(shape[2], 5) * shape[3] * shape[4] * 64) * model_management.dtype_size(dtype) + self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32] + self.handles_tiling = True + self.format_encoded = self.first_stage_model.comfy_format_encoded + self.downscale_ratio = (lambda a: max(0, math.floor((a + 3) / 4)), 8, 8) + self.downscale_index_formula = (4, 8, 8) + self.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8) + self.upscale_index_formula = (4, 8, 8) + self.process_input = lambda image: image * 2.0 - 1.0 + self.crop_input = False elif "decoder.conv_in.weight" in sd: if sd['decoder.conv_in.weight'].shape[1] == 64: ddconfig = {"block_out_channels": [128, 256, 512, 512, 1024, 1024], "in_channels": 3, "out_channels": 3, "num_res_blocks": 2, "ffactor_spatial": 32, "downsample_match_channel": True, "upsample_match_channel": True} @@ -1005,6 +1032,10 @@ class VAE: decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype()) return self.process_output(comfy.utils.tiled_scale_multidim(samples, decode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.upscale_ratio, out_channels=self.output_channels, index_formulas=self.upscale_index_formula, output_device=self.output_device)) + def _decode_tiled_owned(self, samples, **kwargs): + out = self.first_stage_model.decode_tiled(samples.to(self.vae_dtype).to(self.device), **kwargs) + return self.process_output(out.to(device=self.output_device, dtype=self.vae_output_dtype(), copy=True)) + def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64): steps = pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap) steps += pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap) @@ -1041,6 +1072,25 @@ class VAE: encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype()) return comfy.utils.tiled_scale_multidim(samples, encode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.downscale_ratio, out_channels=self.latent_channels, downscale=True, index_formulas=self.downscale_index_formula, output_device=self.output_device) + def _encode_tiled_owned(self, pixel_samples, **kwargs): + x = self.process_input(pixel_samples).to(self.vae_dtype).to(self.device) + out = self.first_stage_model.encode_tiled(x, **kwargs) + return out.to(device=self.output_device, dtype=self.vae_output_dtype()) + + def _owned_tiled_args(self, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None): + args = {} + if tile_x is not None: + args["tile_x"] = tile_x + if tile_y is not None: + args["tile_y"] = tile_y + if overlap is not None: + args["overlap"] = overlap + if tile_t is not None: + args["tile_t"] = tile_t + if overlap_t is not None: + args["overlap_t"] = overlap_t + return args + def decode(self, samples_in, vae_options={}): self.throw_exception_if_invalid() pixel_samples = None @@ -1088,11 +1138,19 @@ class VAE: if dims == 1 or self.extra_1d_channel is not None: pixel_samples = self.decode_tiled_1d(samples_in) elif dims == 2: - pixel_samples = self.decode_tiled_(samples_in) + if self.handles_tiling: + tile = 256 // self.spacial_compression_decode() + overlap = tile // 4 + pixel_samples = self._decode_tiled_owned(samples_in, tile_x=tile, tile_y=tile, overlap=overlap) + else: + pixel_samples = self.decode_tiled_(samples_in) elif dims == 3: tile = 256 // self.spacial_compression_decode() overlap = tile // 4 - pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap)) + if self.handles_tiling: + pixel_samples = self._decode_tiled_owned(samples_in, tile_x=tile, tile_y=tile, overlap=overlap) + else: + pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap)) pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1) return pixel_samples @@ -1111,7 +1169,9 @@ class VAE: args["overlap"] = overlap with model_management.cuda_device_context(self.device): - if dims == 1 or self.extra_1d_channel is not None: + if self.handles_tiling and dims in (2, 3): + output = self._decode_tiled_owned(samples, **self._owned_tiled_args(tile_x, tile_y, overlap, tile_t, overlap_t)) + elif dims == 1 or self.extra_1d_channel is not None: args.pop("tile_y") output = self.decode_tiled_1d(samples, **args) elif dims == 2: @@ -1172,12 +1232,17 @@ class VAE: if self.latent_dim == 3: tile = 256 overlap = tile // 4 - samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap)) + if self.handles_tiling: + samples = self._encode_tiled_owned(pixel_samples, tile_x=tile, tile_y=tile, overlap=overlap) + else: + samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap)) elif self.latent_dim == 1 or self.extra_1d_channel is not None: samples = self.encode_tiled_1d(pixel_samples) else: samples = self.encode_tiled_(pixel_samples) + if self.format_encoded is not None: + samples = self.format_encoded(samples) return samples def encode_tiled(self, pixel_samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None): @@ -1185,7 +1250,7 @@ class VAE: pixel_samples = self.vae_encode_crop_pixels(pixel_samples) dims = self.latent_dim pixel_samples = pixel_samples.movedim(-1, 1) - if dims == 3: + if dims == 3 and pixel_samples.ndim < 5: if not self.not_video: pixel_samples = pixel_samples.movedim(1, 0).unsqueeze(0) else: @@ -1209,21 +1274,27 @@ class VAE: elif dims == 2: samples = self.encode_tiled_(pixel_samples, **args) elif dims == 3: - if tile_t is not None: - tile_t_latent = max(2, self.downscale_ratio[0](tile_t)) + if self.handles_tiling: + samples = self._encode_tiled_owned(pixel_samples, **self._owned_tiled_args(tile_x, tile_y, overlap, tile_t, overlap_t)) else: - tile_t_latent = 9999 - args["tile_t"] = self.upscale_ratio[0](tile_t_latent) + if tile_t is not None: + tile_t_latent = max(2, self.downscale_ratio[0](tile_t)) + else: + tile_t_latent = 9999 + args["tile_t"] = self.upscale_ratio[0](tile_t_latent) - if overlap_t is None: - args["overlap"] = (1, overlap, overlap) - else: - args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), overlap, overlap) - maximum = pixel_samples.shape[2] - maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum)) + spatial_overlap = overlap if overlap is not None else 64 + if overlap_t is None: + args["overlap"] = (1, spatial_overlap, spatial_overlap) + else: + args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), spatial_overlap, spatial_overlap) + maximum = pixel_samples.shape[2] + maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum)) - samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args) + samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args) + if self.format_encoded is not None: + samples = self.format_encoded(samples) return samples def get_sd(self): @@ -1247,6 +1318,11 @@ class VAE: except: return None + def is_dynamic(self): + # A VAE built from a state dict with no detectable VAE weights returns early + # from __init__ ("No VAE weights detected") before self.patcher is assigned. + patcher = getattr(self, "patcher", None) + return patcher is not None and patcher.is_dynamic() class StyleModel: def __init__(self, model, device="cpu"): @@ -1298,6 +1374,9 @@ class CLIPType(Enum): COGVIDEOX = 27 LENS = 28 PIXELDIT = 29 + IDEOGRAM4 = 30 + BOOGU = 31 + KREA2 = 32 @@ -1351,6 +1430,8 @@ class TEModel(Enum): GEMMA_4_31B = 31 T5_GEMMA = 32 GPT_OSS_20B = 33 + QWEN3VL_4B = 34 + QWEN3VL_8B = 35 def detect_te_model(sd): @@ -1412,6 +1493,8 @@ def detect_te_model(sd): if weight.shape[0] == 5120: return TEModel.QWEN35_27B return TEModel.QWEN35_2B + if "model.visual.deepstack_merger_list.0.norm.weight" in sd: # DeepStack is unique to Qwen3-VL + return TEModel.QWEN3VL_4B if sd["model.visual.merger.linear_fc2.weight"].shape[0] == 2560 else TEModel.QWEN3VL_8B if "model.layers.0.post_attention_layernorm.weight" in sd: weight = sd['model.layers.0.post_attention_layernorm.weight'] if 'model.layers.0.self_attn.q_norm.weight' in sd: @@ -1596,8 +1679,12 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip clip_target.clip = comfy.text_encoders.ovis.te(**llama_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.ovis.OvisTokenizer elif te_model == TEModel.QWEN3_8B: - clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type="qwen3_8b") - clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B + if clip_type == CLIPType.IDEOGRAM4: + clip_target.clip = comfy.text_encoders.ideogram4.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.ideogram4.Ideogram4Tokenizer + else: + clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type="qwen3_8b") + clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B elif te_model == TEModel.JINA_CLIP_2: clip_target.clip = comfy.text_encoders.jina_clip_2.JinaClip2TextModelWrapper clip_target.tokenizer = comfy.text_encoders.jina_clip_2.JinaClip2TokenizerWrapper @@ -1606,6 +1693,28 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip qwen35_type = {TEModel.QWEN35_08B: "qwen35_08b", TEModel.QWEN35_2B: "qwen35_2b", TEModel.QWEN35_4B: "qwen35_4b", TEModel.QWEN35_9B: "qwen35_9b", TEModel.QWEN35_27B: "qwen35_27b"}[te_model] clip_target.clip = comfy.text_encoders.qwen35.te(**llama_detect(clip_data), model_type=qwen35_type) clip_target.tokenizer = comfy.text_encoders.qwen35.tokenizer(model_type=qwen35_type) + elif te_model in (TEModel.QWEN3VL_4B, TEModel.QWEN3VL_8B): + if clip_type == CLIPType.IDEOGRAM4 and te_model == TEModel.QWEN3VL_8B: # Ideogram4 reuses the full Qwen3-VL-8B (13-layer tap for conditioning + multimodal generate). + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.ideogram4.te_qwen3vl(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.ideogram4.Ideogram4Qwen3VLTokenizer + elif clip_type == CLIPType.BOOGU and te_model == TEModel.QWEN3VL_8B: # Boogu-Image: full Qwen3-VL-8B, last hidden state, no-think template. + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.boogu.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.boogu.BooguTokenizer + elif clip_type == CLIPType.KREA2 and te_model == TEModel.QWEN3VL_4B: # Krea2: full Qwen3-VL-4B (12-layer tap for conditioning + multimodal generate). + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.krea2.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.krea2.Krea2Tokenizer + elif clip_type in (CLIPType.FLUX, CLIPType.FLUX2): # Flux2 Klein reuses the Qwen3-VL LM (3-layer tap -> 12288); visual unused. + klein_model_type = "qwen3_8b" if te_model == TEModel.QWEN3VL_8B else "qwen3_4b" + clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type=klein_model_type) + clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B if te_model == TEModel.QWEN3VL_8B else comfy.text_encoders.flux.KleinTokenizer + else: + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model] + clip_target.clip = comfy.text_encoders.qwen3vl.te(**llama_detect(clip_data), model_type=qwen3vl_type) + clip_target.tokenizer = comfy.text_encoders.qwen3vl.tokenizer(model_type=qwen3vl_type) elif te_model == TEModel.QWEN3_06B: clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizer @@ -1853,7 +1962,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_c manual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes) else: manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes) - model_config.set_inference_dtype(unet_dtype, manual_cast_dtype) + model_config.set_inference_dtype(unet_dtype, manual_cast_dtype, device=load_device) if model_config.clip_vision_prefix is not None: if output_clipvision: @@ -1994,7 +2103,7 @@ def load_diffusion_model_state_dict(sd, model_options={}, metadata=None, disable manual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes) else: manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes) - model_config.set_inference_dtype(unet_dtype, manual_cast_dtype) + model_config.set_inference_dtype(unet_dtype, manual_cast_dtype, device=load_device) if custom_operations is not None: model_config.custom_operations = custom_operations diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 897186bba..f0fdf1aa5 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -543,18 +543,24 @@ class SDTokenizer: def _try_get_embedding(self, embedding_name:str): ''' Takes a potential embedding name and tries to retrieve it. - Returns a Tuple consisting of the embedding and any leftover string, embedding can be None. + Returns a Tuple consisting of the embedding, the cleaned embedding name, and any leftover string, embedding can be None. ''' split_embed = embedding_name.split() embedding_name = split_embed[0] leftover = ' '.join(split_embed[1:]) + + match = re.search(r'[<\[]', embedding_name) + if match is not None: + leftover = embedding_name[match.start():] + (" " + leftover if leftover else "") + embedding_name = embedding_name[:match.start()] + embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key) if embed is None: stripped = embedding_name.strip(',') if len(stripped) < len(embedding_name): embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key) - return (embed, "{} {}".format(embedding_name[len(stripped):], leftover)) - return (embed, leftover) + return (embed, embedding_name, "{} {}".format(embedding_name[len(stripped):], leftover)) + return (embed, embedding_name, leftover) def pad_tokens(self, tokens, amount): if self.pad_left: @@ -585,7 +591,7 @@ class SDTokenizer: tokens = [] for weighted_segment, weight in parsed_weights: to_tokenize = unescape_important(weighted_segment) - split = re.split(' {0}|\n{0}'.format(self.embedding_identifier), to_tokenize) + split = re.split(r'(?<=\s){}'.format(re.escape(self.embedding_identifier)), to_tokenize) to_tokenize = [split[0]] for i in range(1, len(split)): to_tokenize.append("{}{}".format(self.embedding_identifier, split[i])) @@ -595,7 +601,7 @@ class SDTokenizer: # if we find an embedding, deal with the embedding if word.startswith(self.embedding_identifier) and self.embedding_directory is not None: embedding_name = word[len(self.embedding_identifier):].strip('\n') - embed, leftover = self._try_get_embedding(embedding_name) + embed, embedding_name, leftover = self._try_get_embedding(embedding_name) if embed is None: logging.warning(f"warning, embedding:{embedding_name} does not exist, ignoring") else: diff --git a/comfy/supported_models.py b/comfy/supported_models.py index 0872b0e27..b82e4178f 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -24,6 +24,9 @@ import comfy.text_encoders.qwen_image import comfy.text_encoders.hunyuan_image import comfy.text_encoders.kandinsky5 import comfy.text_encoders.z_image +import comfy.text_encoders.ideogram4 +import comfy.text_encoders.boogu +import comfy.text_encoders.krea2 import comfy.text_encoders.anima import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image @@ -1449,6 +1452,17 @@ class WAN21_SCAIL(WAN21_T2V): out = model_base.WAN21_SCAIL(self, image_to_video=False, device=device) return out + +class WAN21_SCAIL2(WAN21_T2V): + unet_config = { + "image_model": "wan2.1", + "model_type": "scail2", + } + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.WAN21_SCAIL2(self, image_to_video=False, device=device) + return out + class WAN22_WanDancer(WAN21_T2V): unet_config = { "image_model": "wan2.1", @@ -1671,6 +1685,40 @@ class Chroma(supported_models_base.BASE): t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref)) return supported_models_base.ClipTarget(comfy.text_encoders.pixart_t5.PixArtTokenizer, comfy.text_encoders.pixart_t5.pixart_te(**t5_detect)) +class SeedVR2(supported_models_base.BASE): + unet_config = { + "image_model": "seedvr2" + } + unet_extra_config = {} + required_keys = { + "{}positive_conditioning", + "{}negative_conditioning", + } + latent_format = comfy.latent_formats.SeedVR2 + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32] + sampling_settings = { + "shift": 1.0, + } + + def set_inference_dtype(self, dtype, manual_cast_dtype, device=None): + if ( + dtype == torch.float16 + and manual_cast_dtype is None + and comfy.model_management.should_use_bf16(device) + ): + manual_cast_dtype = torch.bfloat16 + super().set_inference_dtype(dtype, manual_cast_dtype, device=device) + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.SeedVR2(self, device=device) + return out + + def clip_target(self, state_dict={}): + return None + class ChromaRadiance(Chroma): unet_config = { "image_model": "chroma_radiance", @@ -1746,6 +1794,94 @@ class Omnigen2(supported_models_base.BASE): hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_3b.transformer.".format(pref)) return supported_models_base.ClipTarget(comfy.text_encoders.omnigen2.Omnigen2Tokenizer, comfy.text_encoders.omnigen2.te(**hunyuan_detect)) +class Boogu(Omnigen2): + unet_config = { + "image_model": "boogu", + } + + sampling_settings = { + "multiplier": 1.0, + "shift": 3.16, + } + + memory_usage_factor = 2.15 + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.Boogu(self, device=device) + return out + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_8b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.boogu.BooguTokenizer, comfy.text_encoders.boogu.te(**hunyuan_detect)) + +class Ideogram4(supported_models_base.BASE): + unet_config = { + "image_model": "ideogram4", + } + + sampling_settings = { + "multiplier": 1.0, + "shift": 1.0, + } + + memory_usage_factor = 11.6 + + unet_extra_config = { + "num_attention_heads": 18, + "attention_head_dim": 256, + "intermediate_size": 12288, + "adaln_dim": 512, + "llm_features_dim": 53248, + "rope_theta": 5000000, + "mrope_section": [24, 20, 20], + "norm_eps": 1e-5, + } + latent_format = latent_formats.Flux2 + + supported_inference_dtypes = [torch.bfloat16, torch.float32] + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.Ideogram4(self, device=device) + return out + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_8b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.ideogram4.Ideogram4Tokenizer, comfy.text_encoders.ideogram4.te(**hunyuan_detect)) + + +class Krea2(supported_models_base.BASE): + unet_config = { + "image_model": "krea2", + } + + sampling_settings = { + "multiplier": 1.0, + "shift": 1.15, + } + + memory_usage_factor = 2.2 + + latent_format = latent_formats.Wan21 + + supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32] + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.Krea2(self, device=device) + return out + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.krea2.Krea2Tokenizer, comfy.text_encoders.krea2.te(**hunyuan_detect)) + class QwenImage(supported_models_base.BASE): unet_config = { "image_model": "qwen_image", @@ -2006,6 +2142,23 @@ class RT_DETR_v4(supported_models_base.BASE): return None +class DepthAnything3(supported_models_base.BASE): + unet_config = { + "image_model": "DepthAnything3", + } + + # Mono path: no num_heads / num_head_channels needed. + unet_extra_config = {} + + supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32] + + def get_model(self, state_dict, prefix="", device=None): + return model_base.DepthAnything3(self, device=device) + + def clip_target(self, state_dict={}): + return None + + class ErnieImage(supported_models_base.BASE): unet_config = { "image_model": "ernie", @@ -2220,6 +2373,7 @@ models = [ WAN22_Animate, WAN21_FlowRVS, WAN21_SCAIL, + WAN21_SCAIL2, WAN22_WanDancer, Hunyuan3Dv2mini, Hunyuan3Dv2, @@ -2228,11 +2382,15 @@ models = [ HiDream, HiDreamO1, Chroma, + SeedVR2, ChromaRadiance, ACEStep, ACEStep15, Omnigen2, + Boogu, QwenImage, + Ideogram4, + Krea2, Flux2, Lens, Kandinsky5Image, @@ -2246,4 +2404,5 @@ models = [ CogVideoX_I2V, CogVideoX_T2V, SVD_img2vid, + DepthAnything3, ] diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.py index 0e7a829ba..e3a8e131f 100644 --- a/comfy/supported_models_base.py +++ b/comfy/supported_models_base.py @@ -54,13 +54,13 @@ class BASE: optimizations = {"fp8": False} @classmethod - def matches(s, unet_config, state_dict=None): + def matches(s, unet_config, state_dict=None, unet_key_prefix=""): for k in s.unet_config: if k not in unet_config or s.unet_config[k] != unet_config[k]: return False if state_dict is not None: for k in s.required_keys: - if k not in state_dict: + if k.format(unet_key_prefix) not in state_dict: return False return True @@ -115,7 +115,7 @@ class BASE: replace_prefix = {"": self.vae_key_prefix[0]} return utils.state_dict_prefix_replace(state_dict, replace_prefix) - def set_inference_dtype(self, dtype, manual_cast_dtype): + def set_inference_dtype(self, dtype, manual_cast_dtype, device=None): self.unet_config['dtype'] = dtype self.manual_cast_dtype = manual_cast_dtype diff --git a/comfy/text_encoders/boogu.py b/comfy/text_encoders/boogu.py new file mode 100644 index 000000000..d9de92f10 --- /dev/null +++ b/comfy/text_encoders/boogu.py @@ -0,0 +1,58 @@ +"""Boogu-Image text encoder: full Qwen3-VL-8B, last hidden state (4096-dim). + +Boogu uses the final hidden state of Qwen3-VL as the per-token instruction feature +(num_instruction_feature_layers=1, reduce_type=mean -> just the last layer). +The model itself is the standard Qwen3-VL TE, only the chat template differs +(a fixed system prompt and no block). +""" + +import comfy.text_encoders.qwen3vl +from comfy import sd1_clip + + +# System prompts from the reference pipeline (pipeline_boogu.py). +# T2I (non-empty instruction, no image) uses the helpful-assistant prompt +# everything else (the CFG negative / "drop" condition, and any image case) uses the TI2I "describe" prompt. +BOOGU_T2I_SYSTEM = "You are a helpful assistant that generates high-quality images based on user instructions. The instructions are as follows." +BOOGU_DROP_SYSTEM = "Describe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate." + + +class BooguTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_8b") + # apply_chat_template without add_generation_prompt + self.llama_template = "<|im_start|>system\n" + BOOGU_T2I_SYSTEM + "<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + self.llama_template_images = "<|im_start|>system\n" + BOOGU_DROP_SYSTEM + "<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n" + # Reference SYSTEM_PROMPT_DROP: used for the empty negative/uncond instruction. + self.llama_template_drop = "<|im_start|>system\n" + BOOGU_DROP_SYSTEM + "<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): + if llama_template is None and len(images) == 0 and text.strip() == "": + llama_template = self.llama_template_drop + # Boogu conditions on the no-think template; thinking=True drops the empty block qwen3vl adds by default. + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) + + +class BooguQwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_8b"): + super().__init__(device=device, dtype=dtype, attention_mask=attention_mask, model_options=model_options, model_type=model_type) + # apply the final RMSNorm to the tapped last layer + self.layer_norm_hidden_state = True + + +class BooguTEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + clip_model = lambda **kw: BooguQwen3VLClipModel(**kw, model_type="qwen3vl_8b") + super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=clip_model, model_options=model_options) + + +def te(dtype_llama=None, llama_quantization_metadata=None): + class BooguTEModel_(BooguTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options) + return BooguTEModel_ diff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.py index f050061ed..0bba8341b 100644 --- a/comfy/text_encoders/gemma4.py +++ b/comfy/text_encoders/gemma4.py @@ -1088,7 +1088,7 @@ class Gemma4_Tokenizer(): h, w = samples.shape[2], samples.shape[3] patch_size = 16 pooling_k = 3 - max_soft_tokens = 70 if is_video else 280 # video uses smaller token budget per frame + max_soft_tokens = kwargs.get("max_soft_tokens", 70 if is_video else 280) max_patches = max_soft_tokens * pooling_k * pooling_k target_px = max_patches * patch_size * patch_size factor = (target_px / (h * w)) ** 0.5 diff --git a/comfy/text_encoders/gpt_oss.py b/comfy/text_encoders/gpt_oss.py index d596ef9a0..066796b6a 100644 --- a/comfy/text_encoders/gpt_oss.py +++ b/comfy/text_encoders/gpt_oss.py @@ -12,7 +12,7 @@ import torch.nn.functional as F import comfy.ops from comfy import sd1_clip -from comfy.ldm.modules.attention import TORCH_HAS_GQA, optimized_attention_for_device +from comfy.ldm.modules.attention import optimized_attention_for_device from comfy.text_encoders.llama import RMSNorm, apply_rope @@ -110,10 +110,6 @@ def _attention_with_sinks(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, sin putting the sink logit in the mask at that column. """ - if num_kv_groups > 1 and not TORCH_HAS_GQA: - k = k.repeat_interleave(num_kv_groups, dim=1) - v = v.repeat_interleave(num_kv_groups, dim=1) - B, _, S_q, D = q.shape H_kv = k.shape[1] S_kv = k.shape[-2] diff --git a/comfy/text_encoders/ideogram4.py b/comfy/text_encoders/ideogram4.py new file mode 100644 index 000000000..151b43c53 --- /dev/null +++ b/comfy/text_encoders/ideogram4.py @@ -0,0 +1,120 @@ +"""Ideogram 4 text encoder: Qwen3-VL-8B language model, 13-layer tap. + +Ideogram 4 conditions on the concatenation of hidden states from 13 layers of +Qwen3-VL (layers 0,3,...,33,35), giving a 4096*13 = 53248-dim feature per token. +""" + +import os + +from transformers import Qwen2Tokenizer + +import comfy.text_encoders.llama +import comfy.text_encoders.qwen3vl +from comfy import sd1_clip + +# Reference taps outputs of layers (0,3,...,35); comfy captures layer inputs, offset by +1. +IDEOGRAM4_TAP_LAYERS = [1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 36] + + +class Qwen3VLTokenizer(sd1_clip.SDTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer") + super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, + embedding_size=4096, embedding_key='qwen3vl_8b', tokenizer_class=Qwen2Tokenizer, + has_start_token=False, has_end_token=False, pad_to_max_length=False, + max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data) + + +class Ideogram4Tokenizer(sd1_clip.SD1Tokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, + name="qwen3vl_8b", tokenizer=Qwen3VLTokenizer) + + self.llama_template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, **kwargs): + if text.startswith('<|im_start|>'): + llama_text = text + elif llama_template is None: + llama_text = self.llama_template.format(text) + else: + llama_text = llama_template.format(text) + return super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs) + + +# Qwen3-VL-8B = 5e6 (vs plain Qwen3-8B's 1e6) +# final_norm/lm_head off -> Ideogram only reads raw tapped hidden states +QWEN3VL_8B_CONFIG = {"rope_theta": 5000000.0, "final_norm": False, "lm_head": False} + + +class Qwen3VL8BModel(sd1_clip.SDClipModel): + def __init__(self, device="cpu", layer="hidden", layer_idx=None, dtype=None, attention_mask=True, model_options={}): + super().__init__(device=device, layer=IDEOGRAM4_TAP_LAYERS, layer_idx=None, + textmodel_json_config=dict(QWEN3VL_8B_CONFIG), + dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, + model_class=comfy.text_encoders.llama.Qwen3_8B, + enable_attention_masks=attention_mask, return_attention_masks=attention_mask, + model_options=model_options) + + +class Ideogram4TEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=Qwen3VL8BModel, model_options=model_options) + + def encode_token_weights(self, token_weight_pairs): + out, pooled, extra = super().encode_token_weights(token_weight_pairs) + b, n, seq, h = out.shape # (B, n_taps=13, seq, 4096) stacked in ascending layer order. + out = out.permute(0, 2, 3, 1).reshape(b, seq, h * n) # (B, seq, 4096*13). permute -> (B, seq, H, taps). + return out, pooled, extra + + +def te(dtype_llama=None, llama_quantization_metadata=None): + class Ideogram4TEModel_(Ideogram4TEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options) + return Ideogram4TEModel_ + + +# Full Qwen3-VL-8B variant with vision + +class Ideogram4Qwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}): + super().__init__(device=device, layer=IDEOGRAM4_TAP_LAYERS, layer_idx=None, dtype=dtype, + attention_mask=attention_mask, model_options=model_options, model_type="qwen3vl_8b") + + +class Ideogram4Qwen3VLTEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=Ideogram4Qwen3VLClipModel, model_options=model_options) + + def encode_token_weights(self, token_weight_pairs): + out, pooled, extra = super().encode_token_weights(token_weight_pairs) + b, n, seq, h = out.shape # (B, n_taps=13, seq, 4096), ascending layer order. + out = out.permute(0, 2, 3, 1).reshape(b, seq, h * n) # (B, seq, 4096*13 = 53248). + return out, pooled, extra + + +class Ideogram4Qwen3VLTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_8b") + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): + # Ideogram 4 conditions on the no-think template; default thinking=True drops the empty think block qwen3vl adds. + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) + + +def te_qwen3vl(dtype_llama=None, llama_quantization_metadata=None): + class Ideogram4Qwen3VLTEModel_(Ideogram4Qwen3VLTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options) + return Ideogram4Qwen3VLTEModel_ diff --git a/comfy/text_encoders/krea2.py b/comfy/text_encoders/krea2.py new file mode 100644 index 000000000..408a03566 --- /dev/null +++ b/comfy/text_encoders/krea2.py @@ -0,0 +1,84 @@ +"""Krea 2 (K2) text encoder: Qwen3-VL-4B, 12-layer tap. + +K2 conditions on a stack of hidden states from 12 layers of Qwen3-VL-4B +(reference taps ``hidden_states[2,5,8,...,35]``), kept as a ``(B, 12, seq, 2560)`` tensor and +consumed by the DiT's internal ``txtfusion`` adapter. Comfy carries conditioning as a 3D tensor, +so the 12-layer stack is flattened to ``(B, seq, 12*2560)`` here and unpacked inside the model. +""" + +import numbers + +import torch + +import comfy.text_encoders.qwen3vl +from comfy import sd1_clip + +# tap k == hidden_states[k] (no offset). +KREA2_TAP_LAYERS = [2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32, 35] + +# Identical system template to Qwen-Image; Krea2 strips the system+user-opening prefix. +KREA2_TEMPLATE = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + + +class Krea2Tokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_4b") + self.llama_template = KREA2_TEMPLATE # conditioning template; image text-gen uses qwen3vl's default image template. + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): + # Krea2 conditions on the no-think template; thinking=True drops the empty block qwen3vl adds. + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) + + +class Krea2Qwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}): + super().__init__(device=device, layer=KREA2_TAP_LAYERS, layer_idx=None, dtype=dtype, + attention_mask=attention_mask, model_options=model_options, model_type="qwen3vl_4b") + + +class Krea2TEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + super().__init__(device=device, dtype=dtype, name="qwen3vl_4b", clip_model=Krea2Qwen3VLClipModel, model_options=model_options) + + def encode_token_weights(self, token_weight_pairs, template_end=-1): + out, pooled, extra = super().encode_token_weights(token_weight_pairs) # out: (B, 12, seq, 2560) + tok_pairs = token_weight_pairs["qwen3vl_4b"][0] + + # Strip the system + user-opening prefix + count_im_start = 0 + if template_end == -1: + for i, v in enumerate(tok_pairs): + elem = v[0] + if not torch.is_tensor(elem) and isinstance(elem, numbers.Integral): + if elem == 151644 and count_im_start < 2: + template_end = i + count_im_start += 1 + if out.shape[2] > (template_end + 3): + if tok_pairs[template_end + 1][0] == 872: # "user" + if tok_pairs[template_end + 2][0] == 198: # "\n" + template_end += 3 + + out = out[:, :, template_end:] + + b, n, seq, h = out.shape + # Flatten the 12-layer axis into the feature dim: (B, seq, 12*2560). Unpacked in the model. + out = out.permute(0, 2, 1, 3).reshape(b, seq, n * h) + + if "attention_mask" in extra: + extra["attention_mask"] = extra["attention_mask"][:, template_end:] + if extra["attention_mask"].sum() == torch.numel(extra["attention_mask"]): + extra.pop("attention_mask") + + return out, pooled, extra + + +def te(dtype_llama=None, llama_quantization_metadata=None): + class Krea2TEModel_(Krea2TEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + if dtype_llama is not None: + dtype = dtype_llama + super().__init__(device=device, dtype=dtype, model_options=model_options) + return Krea2TEModel_ diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 5087228ca..3f98fb0a5 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -251,6 +251,19 @@ class Qwen3_8BConfig: lm_head: bool = True stop_tokens = [151643, 151645] +@dataclass +class Qwen3VL_8BConfig(Qwen3_8BConfig): + max_position_embeddings: int = 262144 + rope_theta: float = 5000000.0 + rope_dims = [24, 20, 20] + interleaved_mrope = True + +@dataclass +class Qwen3VL_4BConfig(Qwen3VL_8BConfig): + hidden_size: int = 2560 + intermediate_size: int = 9728 + lm_head: bool = False # 4B ties word embeddings + @dataclass class Ovis25_2BConfig: vocab_size: int = 151936 @@ -537,10 +550,8 @@ class Attention(nn.Module): xv = xv[:, :, -sliding_window:] attention_mask = attention_mask[..., -sliding_window:] if attention_mask is not None else None - xk = xk.repeat_interleave(self.num_heads // self.num_kv_heads, dim=1) - xv = xv.repeat_interleave(self.num_heads // self.num_kv_heads, dim=1) - - output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True) + gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {} + output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs) return self.o_proj(output), present_key_value class MLP(nn.Module): @@ -703,7 +714,8 @@ class Llama2_(nn.Module): interleaved_mrope=getattr(self.config, "interleaved_mrope", False), device=device) - def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None): + def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, + dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None): if embeds is not None: x = embeds else: @@ -767,6 +779,10 @@ class Llama2_(nn.Module): if current_kv is not None: next_key_values.append(current_kv) + # DeepStack: add per-layer visual features into the first len() decoder layers at image positions (Qwen3-VL) + if deepstack_embeds is not None and i < len(deepstack_embeds): + x[visual_pos_masks] = x[visual_pos_masks] + deepstack_embeds[i].to(x) + if i == intermediate_output: intermediate = x.clone() @@ -860,7 +876,7 @@ class BaseGenerate: torch.empty([batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim], device=device, dtype=execution_dtype), 0)) return past_key_values - def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None): + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None): device = embeds.device if stop_tokens is None: @@ -884,10 +900,18 @@ class BaseGenerate: generated_token_ids = [] pbar = comfy.utils.ProgressBar(max_length) + # MRoPE: prefill uses explicit 3D position_ids, decode continues from the last position + next_pos = int(position_ids[:, -1].max()) + 1 if position_ids is not None else None + # Generation loop current_input_ids = initial_input_ids for step in tqdm(range(max_length), desc="Generating tokens"): - x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids) + # DeepStack visual features are injected on the prefill only; gemma4's forward lacks these kwargs. + extra = {} + if step == 0 and deepstack_embeds is not None: + extra["deepstack_embeds"] = deepstack_embeds + extra["visual_pos_masks"] = visual_pos_masks + x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra) logits = self.logits(x)[:, -1] next_token = self.sample_token(logits, temperature, top_k, top_p, min_p, repetition_penalty, initial_tokens + generated_token_ids, generator, do_sample=do_sample, presence_penalty=presence_penalty) token_id = next_token[0].item() @@ -895,6 +919,9 @@ class BaseGenerate: embeds = self.model.embed_tokens(next_token).to(execution_dtype) current_input_ids = next_token if initial_input_ids is not None else None + if next_pos is not None: # advance MRoPE position for the next (decode) step + position_ids = torch.tensor([[next_pos]], device=device) + next_pos += 1 pbar.update(1) if token_id in stop_tokens: @@ -908,22 +935,41 @@ class BaseGenerate: return torch.argmax(logits, dim=-1, keepdim=True) # Sampling mode - if repetition_penalty != 1.0: - for i in range(logits.shape[0]): - for token_id in set(token_history): - logits[i, token_id] *= repetition_penalty if logits[i, token_id] < 0 else 1/repetition_penalty - - if presence_penalty is not None and presence_penalty != 0.0: - for i in range(logits.shape[0]): - for token_id in set(token_history): - logits[i, token_id] -= presence_penalty + if len(token_history) > 0 and (repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)): + token_ids = torch.tensor(list(set(token_history)), device=logits.device) + token_logits = logits[:, token_ids] + if repetition_penalty != 1.0: + token_logits = torch.where(token_logits < 0, token_logits * repetition_penalty, token_logits / repetition_penalty) + if presence_penalty is not None and presence_penalty != 0.0: + token_logits = token_logits - presence_penalty + logits[:, token_ids] = token_logits if temperature != 1.0: logits = logits / temperature if top_k > 0: - indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] - logits[indices_to_remove] = torch.finfo(logits.dtype).min + top_k = min(top_k, logits.shape[-1]) + logits, top_indices = torch.topk(logits, top_k) + + if min_p > 0.0: + probs_before_filter = torch.nn.functional.softmax(logits, dim=-1) + top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True) + min_threshold = min_p * top_probs + indices_to_remove = probs_before_filter < min_threshold + logits[indices_to_remove] = torch.finfo(logits.dtype).min + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum(torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1) + sorted_indices_to_remove = cumulative_probs > top_p + sorted_indices_to_remove[..., 0] = False + indices_to_remove = torch.zeros_like(logits, dtype=torch.bool) + indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = torch.finfo(logits.dtype).min + + probs = torch.nn.functional.softmax(logits, dim=-1) + next_token = torch.multinomial(probs, num_samples=1, generator=generator) + return top_indices.gather(1, next_token) if min_p > 0.0: probs_before_filter = torch.nn.functional.softmax(logits, dim=-1) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 416ce9d18..304a4357f 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -3,7 +3,6 @@ import torch.nn as nn import torch.nn.functional as F from dataclasses import dataclass, field import os -import math import comfy.model_management from comfy.ldm.modules.attention import optimized_attention_for_device @@ -367,12 +366,8 @@ class GatedAttention(nn.Module): xv = torch.cat((past_value[:, :, :index], xv), dim=2) present_key_value = (xk, xv, index + num_tokens) - # Expand KV heads for GQA - if self.num_heads != self.num_kv_heads: - xk = xk.repeat_interleave(self.num_heads // self.num_kv_heads, dim=1) - xv = xv.repeat_interleave(self.num_heads // self.num_kv_heads, dim=1) - - output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True) + gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {} + output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs) output = output * gate.sigmoid() return self.o_proj(output), present_key_value @@ -563,6 +558,8 @@ class Qwen35VisionModel(nn.Module): for _ in range(config["depth"]) ]) self.merger = Qwen35VisionPatchMerger(self.hidden_size, self.spatial_merge_size, config["out_hidden_size"], device=device, dtype=dtype, ops=ops) + self.deepstack_visual_indexes = [] # DeepStack, per-layer visual features (Qwen3-VL) + self.deepstack_merger_list = None def rot_pos_emb(self, grid_thw): merge_size = self.spatial_merge_size @@ -664,9 +661,14 @@ class Qwen35VisionModel(nn.Module): ).cumsum(dim=0, dtype=torch.int32) cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) optimized_attention = optimized_attention_for_device(x.device, mask=False, small_input=True) - for blk in self.blocks: + deepstack_features = [] + for layer_num, blk in enumerate(self.blocks): x = blk(x, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, optimized_attention=optimized_attention) + if self.deepstack_merger_list is not None and layer_num in self.deepstack_visual_indexes: + deepstack_features.append(self.deepstack_merger_list[self.deepstack_visual_indexes.index(layer_num)](x)) merged = self.merger(x) + if self.deepstack_merger_list is not None: + return merged, deepstack_features return merged # Model Wrapper @@ -690,30 +692,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module): return None, None def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], past_key_values=None): - grid = None - position_ids = None - offset = 0 - for e in embeds_info: - if e.get("type") == "image": - grid = e.get("extra", None) - start = e.get("index") - if position_ids is None: - position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device) - position_ids[:, :start] = torch.arange(0, start, device=embeds.device) - end = e.get("size") + start - len_max = int(grid.max()) // 2 - start_next = len_max + start - position_ids[:, end:] = torch.arange(start_next + offset, start_next + (embeds.shape[1] - end) + offset, device=embeds.device) - position_ids[0, start:end] = start + offset - max_d = int(grid[0][1]) // 2 - position_ids[1, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start] - max_d = int(grid[0][2]) // 2 - position_ids[2, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start] - offset += len_max - (end - start) - - if grid is None: - position_ids = None - + position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values) def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): diff --git a/comfy/text_encoders/qwen3vl.py b/comfy/text_encoders/qwen3vl.py new file mode 100644 index 000000000..7a329d2d6 --- /dev/null +++ b/comfy/text_encoders/qwen3vl.py @@ -0,0 +1,214 @@ +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import Qwen2Tokenizer + +from comfy import sd1_clip +import comfy.text_encoders.qwen_vl +from .qwen35 import Qwen35VisionModel +from .llama import BaseLlama, BaseQwen3, BaseGenerate, Llama2_, Qwen3VL_4BConfig, Qwen3VL_8BConfig + + +QWEN3VL_VISION = { + "qwen3vl_4b": dict(hidden_size=1024, intermediate_size=4096, depth=24, deepstack_visual_indexes=[5, 11, 17]), + "qwen3vl_8b": dict(hidden_size=1152, intermediate_size=4304, depth=27, deepstack_visual_indexes=[8, 16, 24]), +} +QWEN3VL_VISION_COMMON = dict(num_heads=16, patch_size=16, temporal_patch_size=2, in_channels=3, + spatial_merge_size=2, num_position_embeddings=2304) + +QWEN3VL_CONFIGS = {"qwen3vl_4b": Qwen3VL_4BConfig, "qwen3vl_8b": Qwen3VL_8BConfig} + + +class Qwen3VLDeepstackMerger(nn.Module): + # DeepStack merger: postshuffle LayerNorm (applied after spatial merge), unlike the main merger. + def __init__(self, hidden_size, spatial_merge_size, out_hidden_size, device=None, dtype=None, ops=None): + super().__init__() + self.merge_dim = hidden_size * (spatial_merge_size ** 2) + self.norm = ops.LayerNorm(self.merge_dim, eps=1e-6, device=device, dtype=dtype) + self.linear_fc1 = ops.Linear(self.merge_dim, self.merge_dim, device=device, dtype=dtype) + self.linear_fc2 = ops.Linear(self.merge_dim, out_hidden_size, device=device, dtype=dtype) + + def forward(self, x): + x = self.norm(x.view(-1, self.merge_dim)) + return self.linear_fc2(F.gelu(self.linear_fc1(x))) + + +class Qwen3VLVisionModel(Qwen35VisionModel): + # Qwen3.5 vision + DeepStack + def __init__(self, config, device=None, dtype=None, ops=None): + super().__init__(config, device=device, dtype=dtype, ops=ops) + self.deepstack_visual_indexes = config["deepstack_visual_indexes"] + self.deepstack_merger_list = nn.ModuleList([ + Qwen3VLDeepstackMerger(self.hidden_size, self.spatial_merge_size, config["out_hidden_size"], device=device, dtype=dtype, ops=ops) + for _ in self.deepstack_visual_indexes + ]) + + +class Qwen3VL(BaseLlama, BaseQwen3, BaseGenerate, torch.nn.Module): + model_type = "qwen3vl_8b" + + def __init__(self, config_dict, dtype, device, operations): + super().__init__() + config = QWEN3VL_CONFIGS[self.model_type](**config_dict) + self.num_layers = config.num_hidden_layers + self.model = Llama2_(config, device=device, dtype=dtype, ops=operations) + vision_config = {**QWEN3VL_VISION_COMMON, **QWEN3VL_VISION[self.model_type], "out_hidden_size": config.hidden_size} + self.visual = Qwen3VLVisionModel(vision_config, device=device, dtype=dtype, ops=operations) + self.dtype = dtype + + def preprocess_embed(self, embed, device): + if embed["type"] == "image": + # Qwen3-VL normalizes to [-1, 1] (mean/std 0.5), unlike Qwen2.5-VL's CLIP normalization. + image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5]) + merged, deepstack = self.visual(image.to(device, dtype=torch.float32), grid) + return merged, {"grid": grid, "deepstack": deepstack} + return None, None + + def build_image_inputs(self, embeds, embeds_info): + # Returns (position_ids, visual_pos_masks, deepstack) for the prompt + images = sorted([e for e in embeds_info if e.get("type") == "image"], key=lambda e: e["index"]) + if len(images) == 0: + return None, None, None + + device = embeds.device + seq = embeds.shape[1] + position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, seq, device) + + # DeepStack: mask of image positions + per-vision-layer features to inject there. + visual_pos_masks = torch.zeros((1, seq), dtype=torch.bool, device=device) + deepstack = None + for e in images: + start = e["index"] + end = e["size"] + start + visual_pos_masks[0, start:end] = True + ds = e["extra"]["deepstack"] + if deepstack is None: + deepstack = [d for d in ds] + else: + deepstack = [torch.cat([deepstack[i], ds[i]], dim=0) for i in range(len(ds))] + return position_ids, visual_pos_masks, deepstack + + def forward(self, input_ids, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], **kwargs): + position_ids = kwargs.pop("position_ids", None) + visual_pos_masks = kwargs.pop("visual_pos_masks", None) + deepstack_embeds = kwargs.pop("deepstack_embeds", None) + if embeds is not None and position_ids is None: + position_ids, visual_pos_masks, deepstack_embeds = self.build_image_inputs(embeds, embeds_info) + return self.model( + input_ids, + attention_mask=attention_mask, + embeds=embeds, + num_tokens=num_tokens, + intermediate_output=intermediate_output, + final_layer_norm_intermediate=final_layer_norm_intermediate, + dtype=dtype, + position_ids=position_ids, + embeds_info=embeds_info, + visual_pos_masks=visual_pos_masks, + deepstack_embeds=deepstack_embeds, + **kwargs, + ) + + +def _make_qwen3vl_model(model_type): + class Qwen3VL_(Qwen3VL): + pass + Qwen3VL_.model_type = model_type + return Qwen3VL_ + + +class Qwen3VLClipModel(sd1_clip.SDClipModel): + def __init__(self, device="cpu", layer="hidden", layer_idx=-1, dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_8b"): + super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, + dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, + model_class=_make_qwen3vl_model(model_type), enable_attention_masks=attention_mask, + return_attention_masks=attention_mask, model_options=model_options) + + def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0): + if isinstance(tokens, dict): + tokens = next(iter(tokens.values())) + tokens_only = [[t[0] for t in b] for b in tokens] + embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device) + position_ids, visual_pos_masks, deepstack = self.transformer.build_image_inputs(embeds, embeds_info) + return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, + presence_penalty=presence_penalty, position_ids=position_ids, + visual_pos_masks=visual_pos_masks, deepstack_embeds=deepstack) + + +class Qwen3VLTEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen3vl_8b"): + clip_model = lambda **kw: Qwen3VLClipModel(**kw, model_type=model_type) + super().__init__(device=device, dtype=dtype, name=model_type, clip_model=clip_model, model_options=model_options) + + +class Qwen3VLSDTokenizer(sd1_clip.SDTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}, embedding_size=4096, embedding_key="qwen3vl_8b"): + tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer") + super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=embedding_size, embedding_key=embedding_key, tokenizer_class=Qwen2Tokenizer, + has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data) + + +class Qwen3VLTokenizer(sd1_clip.SD1Tokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}, model_type="qwen3vl_8b"): + embedding_size = 2560 if model_type == "qwen3vl_4b" else 4096 + tokenizer = lambda *a, **kw: Qwen3VLSDTokenizer(*a, **kw, embedding_size=embedding_size, embedding_key=model_type) + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name=model_type, tokenizer=tokenizer) + self.llama_template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + self.llama_template_images = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=False, **kwargs): + image = kwargs.get("image", None) + if image is not None and len(images) == 0: + images = [image[i:i + 1] for i in range(image.shape[0])] + + skip_template = text.startswith('<|im_start|>') + if prevent_empty_text and text == '': + text = ' ' + + if skip_template: + llama_text = text + else: + if llama_template is not None: + template = llama_template + elif len(images) == 0: + template = self.llama_template + else: + template = self.llama_template_images + if len(images) > 1: + vision_block = "<|vision_start|><|image_pad|><|vision_end|>" + template = template.replace(vision_block, vision_block * len(images), 1) + llama_text = template.format(text) + if not thinking: # Qwen3 convention: empty think block suppresses reasoning + llama_text += "\n\n\n\n" + + tokens = super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs) + key_name = next(iter(tokens)) + embed_count = 0 + for r in tokens[key_name]: + for i in range(len(r)): + if isinstance(r[i][0], (int, float)) and r[i][0] == 151655: # <|image_pad|> + if len(images) > embed_count: + r[i] = ({"type": "image", "data": images[embed_count], "original_type": "image"},) + r[i][1:] + embed_count += 1 + return tokens + + +def tokenizer(model_type="qwen3vl_8b"): + class Qwen3VLTokenizer_(Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type=model_type) + return Qwen3VLTokenizer_ + + +def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen3vl_8b"): + class Qwen3VLTEModel_(Qwen3VLTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type) + return Qwen3VLTEModel_ diff --git a/comfy/text_encoders/qwen_vl.py b/comfy/text_encoders/qwen_vl.py index 98c350a12..924eb6ad8 100644 --- a/comfy/text_encoders/qwen_vl.py +++ b/comfy/text_encoders/qwen_vl.py @@ -88,6 +88,32 @@ def process_qwen2vl_images( return flatten_patches, image_grid_thw +def qwen2vl_mrope_position_ids(embeds_info, seq_len, device): + # (3, seq_len) T/H/W MRoPE position ids: text runs sequentially, each image span gets its grid positions. + # Returns None when there are no image embeds. `extra` is the image grid_thw, or a dict carrying it under "grid". + position_ids = None + offset = 0 + for e in embeds_info: + if e.get("type") == "image": + extra = e.get("extra", None) + grid = extra["grid"] if isinstance(extra, dict) else extra + start = e.get("index") + if position_ids is None: + position_ids = torch.zeros((3, seq_len), device=device) + position_ids[:, :start] = torch.arange(0, start, device=device) + end = e.get("size") + start + len_max = int(grid.max()) // 2 + start_next = len_max + start + position_ids[:, end:] = torch.arange(start_next + offset, start_next + (seq_len - end) + offset, device=device) + position_ids[0, start:end] = start + offset + max_d = int(grid[0][1]) // 2 + position_ids[1, start:end] = torch.arange(start + offset, start + max_d + offset, device=device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start] + max_d = int(grid[0][2]) // 2 + position_ids[2, start:end] = torch.arange(start + offset, start + max_d + offset, device=device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start] + offset += len_max - (end - start) + return position_ids + + class VisionPatchEmbed(nn.Module): def __init__( self, diff --git a/comfy/utils.py b/comfy/utils.py index 09d783fff..61c2a22dd 100644 --- a/comfy/utils.py +++ b/comfy/utils.py @@ -818,6 +818,44 @@ def z_image_to_diffusers(mmdit_config, output_prefix=""): return key_map +def krea2_to_diffusers(mmdit_config, output_prefix=""): + n_layers = mmdit_config.get("layers", 0) + n_txt_layerwise = 2 # TextFusionTransformer hardcodes 2 layerwise + 2 refiner blocks + n_txt_refiner = 2 + key_map = {} + + def add_block(prefix_to, prefix_from): + block_map = { + "attn.to_q": "attn.wq", "attn.to_k": "attn.wk", "attn.to_v": "attn.wv", + "attn.to_gate": "attn.gate", "attn.to_out.0": "attn.wo", + "attn.to_out": "attn.wo", # some tools drop the ".0" on to_out + "ff.gate": "mlp.gate", "ff.up": "mlp.up", "ff.down": "mlp.down", + } + for d, c in block_map.items(): + key_map["{}.{}.weight".format(prefix_to, d)] = "{}{}.{}.weight".format(output_prefix, prefix_from, c) + + for i in range(n_layers): + add_block("transformer_blocks.{}".format(i), "blocks.{}".format(i)) + for i in range(n_txt_layerwise): + add_block("text_fusion.layerwise_blocks.{}".format(i), "txtfusion.layerwise_blocks.{}".format(i)) + for i in range(n_txt_refiner): + add_block("text_fusion.refiner_blocks.{}".format(i), "txtfusion.refiner_blocks.{}".format(i)) + + MAP_BASIC = [ + ("img_in", "first"), + ("time_embed.linear_1", "tmlp.0"), + ("time_embed.linear_2", "tmlp.2"), + ("time_mod_proj", "tproj.1"), + ("txt_in.linear_1", "txtmlp.1"), + ("txt_in.linear_2", "txtmlp.3"), + ("text_fusion.projector", "txtfusion.projector"), + ("final_layer.linear", "last.linear"), + ] + for d, c in MAP_BASIC: + key_map["{}.weight".format(d)] = "{}{}.weight".format(output_prefix, c) + + return key_map + def repeat_to_batch_size(tensor, batch_size, dim=0): if tensor.shape[dim] > batch_size: return tensor.narrow(dim, 0, batch_size) diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.py index adb5a3144..cb14a5be0 100644 --- a/comfy_api/feature_flags.py +++ b/comfy_api/feature_flags.py @@ -25,6 +25,11 @@ CLI_FEATURE_FLAG_REGISTRY: dict[str, FeatureFlagInfo] = { "default": False, "description": "Show the sign-in button in the frontend even when not signed in", }, + "enable_telemetry": { + "type": "bool", + "default": False, + "description": "Signal the frontend that telemetry collection is enabled", + }, } @@ -95,6 +100,7 @@ def _parse_cli_feature_flags() -> dict[str, Any]: # Default server capabilities _CORE_FEATURE_FLAGS: dict[str, Any] = { "supports_preview_metadata": True, + "supports_model_type_tags": True, "max_upload_size": args.max_upload_size * 1024 * 1024, # Convert MB to bytes "extension": {"manager": {"supports_v4": True}}, "node_replacements": True, diff --git a/comfy_api/latest/_input/video_types.py b/comfy_api/latest/_input/video_types.py index 8fff52c16..e2e99521f 100644 --- a/comfy_api/latest/_input/video_types.py +++ b/comfy_api/latest/_input/video_types.py @@ -27,10 +27,13 @@ class VideoInput(ABC): path: Union[str, IO[bytes]], format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, - metadata: Optional[dict] = None + metadata: Optional[dict] = None, + bit_depth: int | None = None, ): """ Abstract method to save the video input to a file. + + bit_depth selects the encoded bit depth; None keeps the video's native depth. """ pass @@ -83,6 +86,14 @@ class VideoInput(ABC): components = self.get_components() return components.images.shape[2], components.images.shape[1] + def get_bit_depth(self) -> int: + """ + Returns the bit depth of the video (e.g. 8 or 10). + + Default implementation returns 8; subclasses report their real depth. + """ + return 8 + def get_duration(self) -> float: """ Returns the duration of the video in seconds. diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py index 4a12ff9c1..bc95a5b99 100644 --- a/comfy_api/latest/_input_impl/video_types.py +++ b/comfy_api/latest/_input_impl/video_types.py @@ -52,6 +52,12 @@ def get_open_write_kwargs( return open_kwargs +def video_stream_bit_depth(stream) -> int: + if stream is None or stream.format is None or not stream.format.components: + return 8 + return max(component.bits for component in stream.format.components) + + class VideoFromFile(VideoInput): """ Class representing video input from a file. @@ -97,6 +103,13 @@ class VideoFromFile(VideoInput): return stream.width, stream.height raise ValueError(f"No video stream found in file '{self.__file}'") + def get_bit_depth(self) -> int: + if isinstance(self.__file, io.BytesIO): + self.__file.seek(0) # Reset the BytesIO object to the beginning + with av.open(self.__file, mode="r") as container: + video_stream = container.streams.video[0] if len(container.streams.video) > 0 else None + return video_stream_bit_depth(video_stream) + def get_duration(self) -> float: """ Returns the duration of the video in seconds. @@ -257,6 +270,7 @@ class VideoFromFile(VideoInput): image_format = 'gbrpf32le' process_image_format = lambda a: a + align_graph = None audio = None streams = [video_stream] @@ -267,11 +281,18 @@ class VideoFromFile(VideoInput): video_done = False audio_done = True - if len(container.streams.audio): - audio_stream = container.streams.audio[-1] + # Use the last decodable audio stream. Streams FFmpeg has no decoder for have no codec context, + # and decoding their packets crashes the process. (e.g. APAC spatial-audio track in iPhone) + audio_stream = next( + (s for s in reversed(container.streams.audio) if s.codec_context is not None), + None, + ) + if audio_stream is not None: streams += [audio_stream] resampler = av.audio.resampler.AudioResampler(format='fltp') audio_done = False + elif len(container.streams.audio): + logging.warning("No decodable audio stream found in video; ignoring audio.") for packet in container.demux(*streams): if video_done and audio_done: @@ -310,7 +331,28 @@ class VideoFromFile(VideoInput): checked_alpha = True - img = frame.to_ndarray(format=image_format) # shape: (H, W, 4) + # Fix non-deterministic video decode when the video width is not a multiple of 32 + # For non-yuvj pixel formats: most H.264/H.265 video and static images (e.g. lossy WebP via LoadImage) + # Pad both axes to a multiple of 32 and smear the border so the alignment padding never bleeds into the cropped edges + if image_format in ('gbrpf32le', 'gbrapf32le') and frame.width % 32 != 0: + if align_graph is None: + pad_w = ((frame.width + 31) // 32) * 32 + pad_h = ((frame.height + 31) // 32) * 32 + g = av.filter.Graph() + g_src = g.add_buffer(width=frame.width, height=frame.height, + format=frame.format.name, time_base=video_stream.time_base) + g_pad = g.add('pad', f'{pad_w}:{pad_h}:0:0') + g_fill = g.add('fillborders', f'left=0:right={pad_w - frame.width}:top=0:bottom={pad_h - frame.height}:mode=smear') + g_sink = g.add('buffersink') + g_src.link_to(g_pad) + g_pad.link_to(g_fill) + g_fill.link_to(g_sink) + g.configure() + align_graph = (g, g_src, g_sink) + align_graph[1].push(frame) + img = np.ascontiguousarray(align_graph[2].pull().to_ndarray(format=image_format)[:frame.height, :frame.width]) + else: + img = frame.to_ndarray(format=image_format) if frame.rotation != 0: k = int(round(frame.rotation // 90)) img = np.rot90(img, k=k, axes=(0, 1)).copy() @@ -377,25 +419,32 @@ class VideoFromFile(VideoInput): format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, + bit_depth: int | None = None, ): if isinstance(self.__file, io.BytesIO): self.__file.seek(0) # Reset the BytesIO object to the beginning with av.open(self.__file, mode='r') as container: container_format = container.format.name - video_encoding = container.streams.video[0].codec.name if len(container.streams.video) > 0 else None + video_stream = container.streams.video[0] if len(container.streams.video) > 0 else None + video_encoding = video_stream.codec.name if video_stream is not None else None + source_bit_depth = video_stream_bit_depth(video_stream) reuse_streams = True if format != VideoContainer.AUTO and format not in container_format.split(","): reuse_streams = False if codec != VideoCodec.AUTO and codec != video_encoding and video_encoding is not None: reuse_streams = False + if bit_depth is not None and video_encoding is not None and bit_depth != source_bit_depth: + reuse_streams = False if self.__start_time or self.__duration: reuse_streams = False if not reuse_streams: + if bit_depth is None: + bit_depth = source_bit_depth components = self.get_components_internal(container) video = VideoFromComponents(components) return video.save_to( - path, format=format, codec=codec, metadata=metadata + path, format=format, codec=codec, metadata=metadata, bit_depth=bit_depth, ) streams = container.streams @@ -415,10 +464,13 @@ class VideoFromFile(VideoInput): else: output_container.metadata[key] = json.dumps(value) - # Add streams to the new container + # Add streams to the new container. Streams with no codec context cannot be used as an output template. stream_map = {} for stream in streams: if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)): + if stream.codec_context is None: + logging.warning("Skipping %s stream %d with unsupported codec", stream.type, stream.index) + continue out_stream = output_container.add_stream_from_template(template=stream, opaque=True) stream_map[stream] = out_stream @@ -451,8 +503,10 @@ class VideoFromComponents(VideoInput): Class representing video input from tensors. """ - def __init__(self, components: VideoComponents): + def __init__(self, components: VideoComponents, bit_depth: int = 8): self.__components = components + # Tensor components have no inherent bit depth; this is the depth used when encoding. + self.__bit_depth = bit_depth def get_components(self) -> VideoComponents: return VideoComponents( @@ -461,18 +515,26 @@ class VideoFromComponents(VideoInput): frame_rate=self.__components.frame_rate, ) + def get_bit_depth(self) -> int: + return self.__bit_depth + def save_to( self, path: str, format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, + bit_depth: int | None = None, ): """Save the video to a file path or BytesIO buffer.""" if format != VideoContainer.AUTO and format != VideoContainer.MP4: raise ValueError("Only MP4 format is supported for now") if codec != VideoCodec.AUTO and codec != VideoCodec.H264: raise ValueError("Only H264 codec is supported for now") + # None means "use the depth this video was created with" (CreateVideo's choice). + if bit_depth is None: + bit_depth = self.__bit_depth + is_10bit = bit_depth >= 10 extra_kwargs = {} if isinstance(format, VideoContainer) and format != VideoContainer.AUTO: extra_kwargs["format"] = format.value @@ -488,10 +550,11 @@ class VideoFromComponents(VideoInput): frame_rate = Fraction(round(self.__components.frame_rate * 1000), 1000) # Create a video stream + pix_fmt = "yuv420p10le" if is_10bit else "yuv420p" video_stream = output.add_stream('h264', rate=frame_rate) video_stream.width = self.__components.images.shape[2] video_stream.height = self.__components.images.shape[1] - video_stream.pix_fmt = 'yuv420p' + video_stream.pix_fmt = pix_fmt # Create an audio stream audio_sample_rate = 1 @@ -505,9 +568,14 @@ class VideoFromComponents(VideoInput): # Encode video for i, frame in enumerate(self.__components.images): - img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3) - frame = av.VideoFrame.from_ndarray(img, format='rgb24') - frame = frame.reformat(format='yuv420p') # Convert to YUV420P as required by h264 + if is_10bit: + # 16-bit RGB keeps float precision through the conversion to 10-bit YUV. + img = (frame.float() * 65535).clamp(0, 65535).cpu().numpy().astype(np.uint16) # shape: (H, W, 3) + frame = av.VideoFrame.from_ndarray(img, format="rgb48le") + else: + img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3) + frame = av.VideoFrame.from_ndarray(img, format='rgb24') + frame = frame.reformat(format=pix_fmt) packet = video_stream.encode(frame) output.mux(packet) diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.py index a3aa508ce..58e49d8e2 100644 --- a/comfy_api/latest/_io.py +++ b/comfy_api/latest/_io.py @@ -755,6 +755,18 @@ class File3DKSPLAT(ComfyTypeIO): Type = File3D +@comfytype(io_type="FILE_3D_SPLAT_ANY") +class File3DSplatAny(ComfyTypeIO): + """General 3D Gaussian splat file type - accepts any supported splat container (.ply / .spz / .splat / .ksplat).""" + Type = File3D + + +@comfytype(io_type="FILE_3D_POINT_CLOUD_ANY") +class File3DPointCloudAny(ComfyTypeIO): + """General point cloud file type - accepts any supported point cloud container (currently .ply).""" + Type = File3D + + @comfytype(io_type="HOOKS") class Hooks(ComfyTypeIO): if TYPE_CHECKING: @@ -879,6 +891,14 @@ class Tracks(ComfyTypeIO): track_visibility: torch.Tensor Type = TrackDict +@comfytype(io_type="DICT") +class Dict(ComfyTypeIO): + Type = dict + +@comfytype(io_type="ARRAY") +class Array(ComfyTypeIO): + Type = list + @comfytype(io_type="COMFY_MULTITYPED_V3") class MultiType: Type = Any @@ -1267,6 +1287,19 @@ class Color(ComfyTypeIO): def as_dict(self): return super().as_dict() + +@comfytype(io_type="COLORS") +class Colors(ComfyTypeIO): + Type = list[Color.Type] + + class Input(WidgetInput): + def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, + socketless: bool=True, default: list[str]=None, advanced: bool=None): + super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced) + if default is None: + self.default = [] + + @comfytype(io_type="BOUNDING_BOX") class BoundingBox(ComfyTypeIO): class BoundingBoxDict(TypedDict): @@ -1314,6 +1347,20 @@ class Curve(ComfyTypeIO): return d +@comfytype(io_type="BOUNDING_BOXES") +class BoundingBoxes(ComfyTypeIO): + class BoundingBoxWithMetadata(BoundingBox.BoundingBoxDict): + metadata: dict + Type = list[BoundingBoxWithMetadata] + + class Input(WidgetInput): + def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, + socketless: bool=True, default: list[dict]=None, advanced: bool=None): + super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced) + if default is None: + self.default = [] + + @comfytype(io_type="HISTOGRAM") class Histogram(ComfyTypeIO): """A histogram represented as a list of bin counts.""" @@ -1388,7 +1435,8 @@ class V3Data(TypedDict): class HiddenHolder: def __init__(self, unique_id: str, prompt: Any, extra_pnginfo: Any, dynprompt: Any, - auth_token_comfy_org: str, api_key_comfy_org: str, **kwargs): + auth_token_comfy_org: str, api_key_comfy_org: str, + comfy_usage_source: str = None, **kwargs): self.unique_id = unique_id """UNIQUE_ID is the unique identifier of the node, and matches the id property of the node on the client side. It is commonly used in client-server communications (see messages).""" self.prompt = prompt @@ -1401,6 +1449,8 @@ class HiddenHolder: """AUTH_TOKEN_COMFY_ORG is a token acquired from signing into a ComfyOrg account on frontend.""" self.api_key_comfy_org = api_key_comfy_org """API_KEY_COMFY_ORG is an API Key generated by ComfyOrg that allows skipping signing into a ComfyOrg account on frontend.""" + self.comfy_usage_source = comfy_usage_source + """COMFY_USAGE_SOURCE identifies the client that submitted the prompt (e.g. comfyui-frontend, comfy-cli, comfyui-mcp); forwarded to API nodes' upstream requests via the Comfy-Usage-Source header.""" def __getattr__(self, key: str): '''If hidden variable not found, return None.''' @@ -1417,6 +1467,7 @@ class HiddenHolder: dynprompt=d.get(Hidden.dynprompt, None), auth_token_comfy_org=d.get(Hidden.auth_token_comfy_org, None), api_key_comfy_org=d.get(Hidden.api_key_comfy_org, None), + comfy_usage_source=d.get(Hidden.comfy_usage_source, None), ) @classmethod @@ -1439,6 +1490,8 @@ class Hidden(str, Enum): """AUTH_TOKEN_COMFY_ORG is a token acquired from signing into a ComfyOrg account on frontend.""" api_key_comfy_org = "API_KEY_COMFY_ORG" """API_KEY_COMFY_ORG is an API Key generated by ComfyOrg that allows skipping signing into a ComfyOrg account on frontend.""" + comfy_usage_source = "COMFY_USAGE_SOURCE" + """COMFY_USAGE_SOURCE identifies the client that submitted the prompt (e.g. comfyui-frontend, comfy-cli, comfyui-mcp); forwarded to API nodes' upstream requests via the Comfy-Usage-Source header.""" @dataclass @@ -1642,6 +1695,8 @@ class Schema: self.hidden.append(Hidden.auth_token_comfy_org) if Hidden.api_key_comfy_org not in self.hidden: self.hidden.append(Hidden.api_key_comfy_org) + if Hidden.comfy_usage_source not in self.hidden: + self.hidden.append(Hidden.comfy_usage_source) # if is an output_node, will need prompt and extra_pnginfo if self.is_output_node: if Hidden.prompt not in self.hidden: @@ -2336,6 +2391,8 @@ __all__ = [ "File3DSPLAT", "File3DSPZ", "File3DKSPLAT", + "File3DSplatAny", + "File3DPointCloudAny", "Hooks", "HookKeyframes", "TimestepsRange", @@ -2354,6 +2411,8 @@ __all__ = [ "AnyType", "MultiType", "Tracks", + "Dict", + "Array", "Color", # Dynamic Types "MatchType", @@ -2372,6 +2431,8 @@ __all__ = [ "PriceBadgeDepends", "PriceBadge", "BoundingBox", + "BoundingBoxes", + "Colors", "Curve", "Histogram", "Range", diff --git a/comfy_api/latest/_ui.py b/comfy_api/latest/_ui.py index 6592f6b1d..b48713d41 100644 --- a/comfy_api/latest/_ui.py +++ b/comfy_api/latest/_ui.py @@ -285,7 +285,7 @@ class AudioSaveHelper: results = [] for batch_number, waveform in enumerate(audio["waveform"].cpu()): filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) - file = f"{filename_with_batch_num}_{counter:05}_.{format}" + file = f"{filename_with_batch_num}_{counter:05}.{format}" output_path = os.path.join(full_output_folder, file) # Use original sample rate initially diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 9c4cfb9b6..9a7049ea2 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1310,13 +1310,6 @@ class KlingTaskStatus(str, Enum): failed = 'failed' -class KlingTextToVideoModelName(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_6 = 'kling-v1-6' - kling_v2_1_master = 'kling-v2-1-master' - kling_v2_5_turbo = 'kling-v2-5-turbo' - - class KlingVideoGenAspectRatio(str, Enum): field_16_9 = '16:9' field_9_16 = '9:16' @@ -5179,7 +5172,7 @@ class KlingText2VideoRequest(BaseModel): duration: Optional[KlingVideoGenDuration] = '5' external_task_id: Optional[str] = Field(None, description='Customized Task ID') mode: Optional[KlingVideoGenMode] = 'std' - model_name: Optional[KlingTextToVideoModelName] = 'kling-v1' + model_name: Optional[str] = 'kling-v1' negative_prompt: Optional[str] = Field( None, description='Negative text prompt', max_length=2500 ) diff --git a/comfy_api_nodes/apis/bfl.py b/comfy_api_nodes/apis/bfl.py index 2ad651122..4c950da84 100644 --- a/comfy_api_nodes/apis/bfl.py +++ b/comfy_api_nodes/apis/bfl.py @@ -43,6 +43,7 @@ class BFLFluxEraseRequest(BaseModel): "white (255) marks areas to remove, black (0) marks areas to preserve.", ) dilate_pixels: int = Field(10) + seed: int | None = Field(None) output_format: str = Field("png") diff --git a/comfy_api_nodes/apis/bria.py b/comfy_api_nodes/apis/bria.py index e08a519a8..7a98428c3 100644 --- a/comfy_api_nodes/apis/bria.py +++ b/comfy_api_nodes/apis/bria.py @@ -97,3 +97,28 @@ class BriaRemoveVideoBackgroundResult(BaseModel): class BriaRemoveVideoBackgroundResponse(BaseModel): status: str = Field(...) result: BriaRemoveVideoBackgroundResult | None = Field(None) + + +class BriaVideoGreenScreenRequest(BaseModel): + video: str = Field(..., description="Publicly accessible URL of the input video.") + green_shade: str = Field( + default="broadcast_green", + description="Solid chroma-key shade applied behind the foreground " + "(broadcast_green, chroma_green, or blue_screen).", + ) + output_container_and_codec: str = Field(...) + preserve_audio: bool = Field(True) + seed: int = Field(...) + + +class BriaVideoReplaceBackgroundRequest(BaseModel): + video: str = Field(..., description="Publicly accessible URL of the input (foreground) video.") + background_url: str = Field( + ..., + description="Publicly accessible URL of the background image or video to composite behind " + "the foreground. Stretched to the foreground frame; match its aspect ratio for " + "undistorted results.", + ) + output_container_and_codec: str = Field(...) + preserve_audio: bool = Field(True) + seed: int = Field(...) diff --git a/comfy_api_nodes/apis/bytedance.py b/comfy_api_nodes/apis/bytedance.py index 47f24586c..515e124ca 100644 --- a/comfy_api_nodes/apis/bytedance.py +++ b/comfy_api_nodes/apis/bytedance.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Any, Literal from pydantic import BaseModel, Field @@ -17,6 +17,10 @@ class Seedream4Options(BaseModel): max_images: int = Field(15) +class Seedream5OptimizePromptOptions(BaseModel): + thinking: Literal["auto", "enabled", "disabled"] = Field(...) + + class Seedream4TaskCreationRequest(BaseModel): model: str = Field(...) prompt: str = Field(...) @@ -24,10 +28,11 @@ class Seedream4TaskCreationRequest(BaseModel): image: list[str] | None = Field(None, description="Image URLs") size: str = Field(...) seed: int = Field(..., ge=0, le=2147483647) - sequential_image_generation: str = Field("disabled") - sequential_image_generation_options: Seedream4Options = Field(Seedream4Options(max_images=15)) + sequential_image_generation: str | None = Field("disabled") + sequential_image_generation_options: Seedream4Options | None = Field(Seedream4Options(max_images=15)) watermark: bool = Field(False) output_format: str | None = None + optimize_prompt_options: Seedream5OptimizePromptOptions | None = None class ImageTaskCreationResponse(BaseModel): @@ -163,15 +168,31 @@ class SeedanceVirtualLibraryCreateAssetRequest(BaseModel): asset_type: str | None = Field(None, description="BytePlus asset type. Defaults to Image server-side when omitted.") -# Dollars per 1K tokens, keyed by (model_id, has_video_input). +# Dollars per 1K tokens, keyed by (model_id, has_video_input, resolution). SEEDANCE2_PRICE_PER_1K_TOKENS = { - ("dreamina-seedance-2-0-260128", False): 0.007, - ("dreamina-seedance-2-0-260128", True): 0.0043, - ("dreamina-seedance-2-0-fast-260128", False): 0.0056, - ("dreamina-seedance-2-0-fast-260128", True): 0.0033, + ("dreamina-seedance-2-0-260128", False, "480p"): 0.007, + ("dreamina-seedance-2-0-260128", True, "480p"): 0.0043, + ("dreamina-seedance-2-0-260128", False, "720p"): 0.007, + ("dreamina-seedance-2-0-260128", True, "720p"): 0.0043, + ("dreamina-seedance-2-0-260128", False, "1080p"): 0.0077, + ("dreamina-seedance-2-0-260128", True, "1080p"): 0.0047, + ("dreamina-seedance-2-0-260128", False, "4k"): 0.004, + ("dreamina-seedance-2-0-260128", True, "4k"): 0.0024, + ("dreamina-seedance-2-0-fast-260128", False, "480p"): 0.0056, + ("dreamina-seedance-2-0-fast-260128", True, "480p"): 0.0033, + ("dreamina-seedance-2-0-fast-260128", False, "720p"): 0.0056, + ("dreamina-seedance-2-0-fast-260128", True, "720p"): 0.0033, + ("dreamina-seedance-2-0-mini", False, "480p"): 0.0035, + ("dreamina-seedance-2-0-mini", True, "480p"): 0.0021, + ("dreamina-seedance-2-0-mini", False, "720p"): 0.0035, + ("dreamina-seedance-2-0-mini", True, "720p"): 0.0021, } +def seedance2_price_per_1k_tokens(model_id: str, has_video_input: bool, resolution: str) -> float | None: + return SEEDANCE2_PRICE_PER_1K_TOKENS.get((model_id, has_video_input, resolution)) + + RECOMMENDED_PRESETS = [ ("1024x1024 (1:1)", 1024, 1024), ("864x1152 (3:4)", 864, 1152), @@ -245,6 +266,19 @@ _PRESETS_SEEDREAM_4K = [ _CUSTOM_PRESET = [("Custom", None, None)] +_PRESETS_SEEDREAM_2K_PRO = [ + ("(2K) 2048x2048 (1:1)", 2048, 2048), + ("(2K) 1728x2304 (3:4)", 1728, 2304), + ("(2K) 2304x1728 (4:3)", 2304, 1728), + # ("(2K) 2848x1600 (16:9)", 2848, 1600), # 4,556,800 px - temporarily unavailable + # ("(2K) 1600x2848 (9:16)", 1600, 2848), # 4,556,800 px - temporarily unavailable + ("(2K) 1664x2496 (2:3)", 1664, 2496), + ("(2K) 2496x1664 (3:2)", 2496, 1664), + # ("(2K) 3136x1344 (21:9)", 3136, 1344), # 4,214,784 px - temporarily unavailable +] +RECOMMENDED_PRESETS_SEEDREAM_5_PRO = ( + _PRESETS_SEEDREAM_1K + _PRESETS_SEEDREAM_2K_PRO + _CUSTOM_PRESET +) RECOMMENDED_PRESETS_SEEDREAM_5_LITE = ( _PRESETS_SEEDREAM_2K + _PRESETS_SEEDREAM_3K + _PRESETS_SEEDREAM_4K + _CUSTOM_PRESET ) @@ -266,6 +300,10 @@ SEEDANCE2_REF_VIDEO_PIXEL_LIMITS = { "480p": {"min": 409_600, "max": 927_408}, "720p": {"min": 409_600, "max": 927_408}, }, + "dreamina-seedance-2-0-mini": { + "480p": {"min": 409_600, "max": 927_408}, + "720p": {"min": 409_600, "max": 927_408}, + }, } # The time in this dictionary are given for 10 seconds duration. @@ -296,3 +334,36 @@ VIDEO_TASKS_EXECUTION_TIME = { "1080p": 150, }, } + + +class SeedAudioConfig(BaseModel): + format: str = Field(default="mp3") + sample_rate: int = Field(default=24000) + speech_rate: int = Field(default=0) + loudness_rate: int = Field(default=0) + pitch_rate: int = Field(default=0) + + +class SeedAudioReference(BaseModel): + speaker: str | None = Field(default=None) + audio_data: str | None = Field(default=None) + audio_url: str | None = Field(default=None) + image_data: str | None = Field(default=None) + image_url: str | None = Field(default=None) + + +class SeedAudioRequest(BaseModel): + model: str = Field(default="seed-audio-1.0") + text_prompt: str = Field(...) + references: list[SeedAudioReference] | None = Field(default=None) + audio_config: SeedAudioConfig = Field(default_factory=SeedAudioConfig) + watermark: dict[str, Any] = Field(default_factory=dict) + + +class SeedAudioResponse(BaseModel): + audio: str | None = Field(default=None) + url: str | None = Field(default=None) + duration: float | None = Field(default=None) + original_duration: float | None = Field(default=None) + code: int | None = Field(default=None) + message: str | None = Field(default=None) diff --git a/comfy_api_nodes/apis/gemini.py b/comfy_api_nodes/apis/gemini.py index 22879fe18..7b2543270 100644 --- a/comfy_api_nodes/apis/gemini.py +++ b/comfy_api_nodes/apis/gemini.py @@ -108,13 +108,20 @@ class GeminiVideoMetadata(BaseModel): startOffset: GeminiOffset | None = Field(None) +class GeminiThinkingConfig(BaseModel): + includeThoughts: bool | None = Field(None) + thinkingLevel: str = Field(...) + + class GeminiGenerationConfig(BaseModel): - maxOutputTokens: int | None = Field(None, ge=16, le=8192) + maxOutputTokens: int | None = Field(None, ge=16, le=65536) seed: int | None = Field(None) stopSequences: list[str] | None = Field(None) temperature: float | None = Field(None, ge=0.0, le=2.0) topK: int | None = Field(None, ge=1) topP: float | None = Field(None, ge=0.0, le=1.0) + thinkingConfig: GeminiThinkingConfig | None = Field(None) + responseModalities: list[str] | None = Field(None) class GeminiImageOutputOptions(BaseModel): @@ -128,11 +135,6 @@ class GeminiImageConfig(BaseModel): imageOutputOptions: GeminiImageOutputOptions = Field(default_factory=GeminiImageOutputOptions) -class GeminiThinkingConfig(BaseModel): - includeThoughts: bool | None = Field(None) - thinkingLevel: str = Field(...) - - class GeminiImageGenerationConfig(GeminiGenerationConfig): responseModalities: list[str] | None = Field(None) imageConfig: GeminiImageConfig | None = Field(None) diff --git a/comfy_api_nodes/apis/hunyuan3d.py b/comfy_api_nodes/apis/hunyuan3d.py index dad9bc2fa..91f630e81 100644 --- a/comfy_api_nodes/apis/hunyuan3d.py +++ b/comfy_api_nodes/apis/hunyuan3d.py @@ -77,6 +77,7 @@ class To3DUVTaskRequest(BaseModel): class To3DPartTaskRequest(BaseModel): File: TaskFile3DInput = Field(...) + EnableStagedGeneration: bool | None = Field(None) class TextureEditImageInfo(BaseModel): diff --git a/comfy_api_nodes/apis/ideogram.py b/comfy_api_nodes/apis/ideogram.py index 737e18e3b..ee3256e96 100644 --- a/comfy_api_nodes/apis/ideogram.py +++ b/comfy_api_nodes/apis/ideogram.py @@ -33,53 +33,6 @@ class IdeogramColorPalette( ) -class ImageRequest(BaseModel): - aspect_ratio: Optional[str] = Field( - None, - description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", - ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' - ) - magic_prompt_option: Optional[str] = Field( - None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." - ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") - negative_prompt: Optional[str] = Field( - None, - description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', - ) - num_images: Optional[int] = Field( - 1, - description='Optional. Number of images to generate (1-8). Defaults to 1.', - ge=1, - le=8, - ) - prompt: str = Field( - ..., description='Required. The prompt to use to generate the image.' - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) - seed: Optional[int] = Field( - None, - description='Optional. A number between 0 and 2147483647.', - ge=0, - le=2147483647, - ) - style_type: Optional[str] = Field( - None, - description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", - ) - - -class IdeogramGenerateRequest(BaseModel): - image_request: ImageRequest = Field( - ..., description='The image generation request parameters.' - ) - - class Datum(BaseModel): is_image_safe: Optional[bool] = Field( None, description='Indicates whether the image is considered safe.' @@ -113,20 +66,6 @@ class StyleCode(RootModel[str]): root: str = Field(..., pattern='^[0-9A-Fa-f]{8}$') -class Datum1(BaseModel): - is_image_safe: Optional[bool] = None - prompt: Optional[str] = None - resolution: Optional[str] = None - seed: Optional[int] = None - style_type: Optional[str] = None - url: Optional[str] = None - - -class IdeogramV3IdeogramResponse(BaseModel): - created: Optional[datetime] = None - data: Optional[List[Datum1]] = None - - class RenderingSpeed1(str, Enum): TURBO = 'TURBO' DEFAULT = 'DEFAULT' @@ -290,3 +229,19 @@ class IdeogramV3Request(BaseModel): None, description='Optional masks for character reference images. When provided, must match the number of character_reference_images. Each mask should be a grayscale image of the same dimensions as the corresponding character reference image. The images should be in JPEG, PNG or WebP format.' ) + + +class IdeogramV4Request(BaseModel): + text_prompt: str | None = Field( + None, + description="Natural-language prompt; Magic Prompt is applied automatically. " + "Supply exactly one of text_prompt or json_prompt.", + ) + json_prompt: dict[str, Any] | None = Field( + None, + description="Structured V4 prompt object consumed directly (disables Magic Prompt). " + "Supply exactly one of text_prompt or json_prompt.", + ) + resolution: str | None = Field(None, description="Output resolution in WIDTHxHEIGHT (e.g. '2048x2048').") + rendering_speed: str | None = Field(None, description="Rendering speed: 'TURBO', 'DEFAULT', or 'QUALITY'.") + enable_copyright_detection: bool | None = Field(None, description="Opt into post-generation copyright detection.") diff --git a/comfy_api_nodes/apis/kling.py b/comfy_api_nodes/apis/kling.py index fe0f97cb3..2c98c23b7 100644 --- a/comfy_api_nodes/apis/kling.py +++ b/comfy_api_nodes/apis/kling.py @@ -149,3 +149,59 @@ class MotionControlRequest(BaseModel): character_orientation: str = Field(...) mode: str = Field(..., description="'pro' or 'std'") model_name: str = Field(...) + + +class Kling3TurboSettings(BaseModel): + resolution: str = Field("720p", description="'720p' or '1080p'") + aspect_ratio: str | None = Field(None, description="'16:9'/'9:16'/'1:1'; text-to-video only") + duration: int = Field(5, description="3-15 second") + + +class Kling3TurboText2VideoRequest(BaseModel): + prompt: str = Field(..., description="<=3072 chars; may use multi-shot 'shot n, m, words; ...'") + settings: Kling3TurboSettings | None = Field(None) + + +class Kling3TurboContent(BaseModel): + type: str = Field(..., description="'prompt' or 'first_frame'") + text: str | None = Field(None, description="for type=prompt; <=2500 chars") + url: str | None = Field(None, description="for type=first_frame") + + +class Kling3TurboImage2VideoRequest(BaseModel): + contents: list[Kling3TurboContent] = Field(..., description="prompt + first_frame materials") + settings: Kling3TurboSettings | None = Field(None) + + +class Kling3TurboCreateData(BaseModel): + id: str | None = Field(None, description="Task ID") + status: str | None = Field(None) + message: str | None = Field(None) + + +class Kling3TurboCreateResponse(BaseModel): + code: int | None = Field(None) + message: str | None = Field(None) + request_id: str | None = Field(None) + data: Kling3TurboCreateData | None = Field(None) + + +class Kling3TurboOutput(BaseModel): + type: str | None = Field(None, description="'video', 'image', 'audio', ...") + id: str | None = Field(None) + url: str | None = Field(None) + duration: str | None = Field(None) + + +class Kling3TurboTaskData(BaseModel): + id: str | None = Field(None) + status: str | None = Field(None, description="submitted | processing | succeeded | failed") + message: str | None = Field(None) + outputs: list[Kling3TurboOutput] | None = Field(None) + + +class Kling3TurboQueryResponse(BaseModel): + code: int | None = Field(None) + message: str | None = Field(None) + request_id: str | None = Field(None) + data: list[Kling3TurboTaskData] | None = Field(None) diff --git a/comfy_api_nodes/apis/luma.py b/comfy_api_nodes/apis/luma.py index 8c6db2022..2465c3b37 100644 --- a/comfy_api_nodes/apis/luma.py +++ b/comfy_api_nodes/apis/luma.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, Field, confloat class LumaIO: LUMA_REF = "LUMA_REF" LUMA_CONCEPTS = "LUMA_CONCEPTS" + LUMA_RAY32_KEYFRAME = "LUMA_RAY32_KEYFRAME" class LumaReference: @@ -20,13 +21,14 @@ class LumaReference: def create_api_model(self, download_url: str): return LumaImageRef(url=download_url, weight=self.weight) + class LumaReferenceChain: - def __init__(self, first_ref: LumaReference=None): + def __init__(self, first_ref: LumaReference = None): self.refs: list[LumaReference] = [] if first_ref: self.refs.append(first_ref) - def add(self, luma_ref: LumaReference=None): + def add(self, luma_ref: LumaReference = None): self.refs.append(luma_ref) def create_api_model(self, download_urls: list[str], max_refs=4): @@ -124,7 +126,7 @@ def get_luma_concepts(include_none=False): "pull_out", "aerial", "crane_up", - "eye_level" + "eye_level", ] @@ -162,8 +164,8 @@ class LumaVideoModelOutputDuration(str, Enum): class LumaGenerationType(str, Enum): - video = 'video' - image = 'image' + video = "video" + image = "image" class LumaState(str, Enum): @@ -174,86 +176,109 @@ class LumaState(str, Enum): class LumaAssets(BaseModel): - video: Optional[str] = Field(None, description='The URL of the video') - image: Optional[str] = Field(None, description='The URL of the image') - progress_video: Optional[str] = Field(None, description='The URL of the progress video') + video: Optional[str] = Field(None, description="The URL of the video") + image: Optional[str] = Field(None, description="The URL of the image") + progress_video: Optional[str] = Field(None, description="The URL of the progress video") class LumaImageRef(BaseModel): """Used for image gen""" - url: str = Field(..., description='The URL of the image reference') - weight: confloat(ge=0.0, le=1.0) = Field(..., description='The weight of the image reference') + + url: str = Field(..., description="The URL of the image reference") + weight: confloat(ge=0.0, le=1.0) = Field(..., description="The weight of the image reference") class LumaImageReference(BaseModel): """Used for video gen""" - type: Optional[str] = Field('image', description='Input type, defaults to image') - url: str = Field(..., description='The URL of the image') + + type: Optional[str] = Field("image", description="Input type, defaults to image") + url: str = Field(..., description="The URL of the image") class LumaModifyImageRef(BaseModel): - url: str = Field(..., description='The URL of the image reference') - weight: confloat(ge=0.0, le=1.0) = Field(..., description='The weight of the image reference') + url: str = Field(..., description="The URL of the image reference") + weight: confloat(ge=0.0, le=1.0) = Field(..., description="The weight of the image reference") class LumaCharacterRef(BaseModel): - identity0: LumaImageIdentity = Field(..., description='The image identity object') + identity0: LumaImageIdentity = Field(..., description="The image identity object") class LumaImageIdentity(BaseModel): - images: list[str] = Field(..., description='The URLs of the image identity') + images: list[str] = Field(..., description="The URLs of the image identity") class LumaGenerationReference(BaseModel): - type: str = Field('generation', description='Input type, defaults to generation') - id: str = Field(..., description='The ID of the generation') + type: str = Field("generation", description="Input type, defaults to generation") + id: str = Field(..., description="The ID of the generation") class LumaKeyframes(BaseModel): - frame0: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') - frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') + frame0: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description="") + frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description="") class LumaConceptObject(BaseModel): - key: str = Field(..., description='Camera Concept name') + key: str = Field(..., description="Camera Concept name") class LumaImageGenerationRequest(BaseModel): - prompt: str = Field(..., description='The prompt of the generation') - model: LumaImageModel = Field(LumaImageModel.photon_1, description='The image model used for the generation') - aspect_ratio: Optional[LumaAspectRatio] = Field(LumaAspectRatio.ratio_16_9, description='The aspect ratio of the generation') - image_ref: Optional[list[LumaImageRef]] = Field(None, description='List of image reference objects') - style_ref: Optional[list[LumaImageRef]] = Field(None, description='List of style reference objects') - character_ref: Optional[LumaCharacterRef] = Field(None, description='The image identity object') - modify_image_ref: Optional[LumaModifyImageRef] = Field(None, description='The modify image reference object') + prompt: str = Field(..., description="The prompt of the generation") + model: LumaImageModel = Field(LumaImageModel.photon_1, description="The image model used for the generation") + aspect_ratio: Optional[LumaAspectRatio] = Field(LumaAspectRatio.ratio_16_9) + image_ref: Optional[list[LumaImageRef]] = Field(None, description="List of image reference objects") + style_ref: Optional[list[LumaImageRef]] = Field(None, description="List of style reference objects") + character_ref: Optional[LumaCharacterRef] = Field(None, description="The image identity object") + modify_image_ref: Optional[LumaModifyImageRef] = Field(None, description="The modify image reference object") class LumaGenerationRequest(BaseModel): - prompt: str = Field(..., description='The prompt of the generation') - model: LumaVideoModel = Field(LumaVideoModel.ray_2, description='The video model used for the generation') - duration: Optional[LumaVideoModelOutputDuration] = Field(None, description='The duration of the generation') - aspect_ratio: Optional[LumaAspectRatio] = Field(None, description='The aspect ratio of the generation') - resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation') - loop: Optional[bool] = Field(None, description='Whether to loop the video') - keyframes: Optional[LumaKeyframes] = Field(None, description='The keyframes of the generation') - concepts: Optional[list[LumaConceptObject]] = Field(None, description='Camera Concepts to apply to generation') + prompt: str = Field(..., description="The prompt of the generation") + model: LumaVideoModel = Field(LumaVideoModel.ray_2, description="The video model used for the generation") + duration: Optional[LumaVideoModelOutputDuration] = Field(None, description="The duration of the generation") + aspect_ratio: Optional[LumaAspectRatio] = Field(None, description="The aspect ratio of the generation") + resolution: Optional[LumaVideoOutputResolution] = Field(None, description="The resolution of the generation") + loop: Optional[bool] = Field(None, description="Whether to loop the video") + keyframes: Optional[LumaKeyframes] = Field(None, description="The keyframes of the generation") + concepts: Optional[list[LumaConceptObject]] = Field(None, description="Camera Concepts to apply to generation") class LumaGeneration(BaseModel): - id: str = Field(..., description='The ID of the generation') - generation_type: LumaGenerationType = Field(..., description='Generation type, image or video') - state: LumaState = Field(..., description='The state of the generation') - failure_reason: Optional[str] = Field(None, description='The reason for the state of the generation') - created_at: str = Field(..., description='The date and time when the generation was created') - assets: Optional[LumaAssets] = Field(None, description='The assets of the generation') - model: str = Field(..., description='The model used for the generation') - request: Union[LumaGenerationRequest, LumaImageGenerationRequest] = Field(..., description="The request used for the generation") + id: str = Field(..., description="The ID of the generation") + generation_type: LumaGenerationType = Field(..., description="Generation type, image or video") + state: LumaState = Field(..., description="The state of the generation") + failure_reason: Optional[str] = Field(None, description="The reason for the state of the generation") + created_at: str = Field(..., description="The date and time when the generation was created") + assets: Optional[LumaAssets] = Field(None, description="The assets of the generation") + model: str = Field(..., description="The model used for the generation") + request: Union[LumaGenerationRequest, LumaImageGenerationRequest] = Field(...) class Luma2ImageRef(BaseModel): url: str | None = None data: str | None = None media_type: str | None = None + generation_id: str | None = Field(None, description="reference a prior generation (extend / source reuse)") + + +class Luma2VideoEdit(BaseModel): + """Edit controls for Ray 3.2 ``video_edit`` generations.""" + + auto_controls: bool | None = Field(None, description="derive a conditioning schedule from the source (recommended)") + strength: str | None = Field(None, description="'adhere_1' .. 'reimagine_3'; constrained by IO.Combo") + + +class Luma2VideoOptions(BaseModel): + """Ray 3.2 ``video`` output settings (text / image / keyframe / edit / extend).""" + + resolution: str | None = Field(None, description="360p | 540p | 720p | 1080p") + duration: str | None = Field(None, description="5s | 10s") + loop: bool | None = Field(None) + start_frame: Luma2ImageRef | None = Field(None) + end_frame: Luma2ImageRef | None = Field(None) + keyframes: list[Luma2ImageRef] | None = Field(None) + keyframe_indexes: list[int] | None = Field(None) + edit: Luma2VideoEdit | None = Field(None) class Luma2GenerationRequest(BaseModel): @@ -266,6 +291,7 @@ class Luma2GenerationRequest(BaseModel): web_search: bool | None = None image_ref: list[Luma2ImageRef] | None = None source: Luma2ImageRef | None = None + video: Luma2VideoOptions | None = Field(None) class Luma2Generation(BaseModel): @@ -277,3 +303,31 @@ class Luma2Generation(BaseModel): output: list[LumaImageReference] | None = None failure_reason: str | None = None failure_code: str | None = None + + +# --- Ray 3.2 multi-keyframe chain --- + +LUMA_KEYFRAME_MODE_FRACTION = "fraction" # value in [0.0, 1.0] of the output video duration +LUMA_KEYFRAME_MODE_SECONDS = "seconds" # absolute time, in seconds, from the start of the output + + +class LumaRay32KeyframeItem: + """One guide image anchored at a position on the Ray 3.2 output timeline.""" + + def __init__(self, image: torch.Tensor, mode: str, value: float): + self.image = image + self.mode = mode # LUMA_KEYFRAME_MODE_FRACTION | LUMA_KEYFRAME_MODE_SECONDS + self.value = value + + +class LumaRay32KeyframeChain: + def __init__(self): + self.items: list[LumaRay32KeyframeItem] = [] + + def add(self, item: LumaRay32KeyframeItem) -> None: + self.items.append(item) + + def clone(self) -> "LumaRay32KeyframeChain": + c = LumaRay32KeyframeChain() + c.items = list(self.items) + return c diff --git a/comfy_api_nodes/apis/runway.py b/comfy_api_nodes/apis/runway.py index df6f2b845..6878aa6f0 100644 --- a/comfy_api_nodes/apis/runway.py +++ b/comfy_api_nodes/apis/runway.py @@ -67,15 +67,6 @@ class RunwayImageToVideoResponse(BaseModel): id: Optional[str] = Field(None, description='Task ID') -class RunwayTaskStatusEnum(str, Enum): - SUCCEEDED = 'SUCCEEDED' - RUNNING = 'RUNNING' - FAILED = 'FAILED' - PENDING = 'PENDING' - CANCELLED = 'CANCELLED' - THROTTLED = 'THROTTLED' - - class RunwayTaskStatusResponse(BaseModel): createdAt: datetime = Field(..., description='Task creation timestamp') id: str = Field(..., description='Task ID') @@ -86,7 +77,7 @@ class RunwayTaskStatusResponse(BaseModel): ge=0.0, le=1.0, ) - status: RunwayTaskStatusEnum + status: str = Field(..., description="SUCCEEDED, RUNNING, FAILED, PENDING, CANCELLED or THROTTLED") class Model4(str, Enum): @@ -125,3 +116,144 @@ class RunwayTextToImageRequest(BaseModel): class RunwayTextToImageResponse(BaseModel): id: Optional[str] = Field(None, description='Task ID') + + +class RunwayAleph2IO: + """Custom socket types for chaining Aleph2 guidance images.""" + + KEYFRAME = "RUNWAY_ALEPH2_KEYFRAME" + PROMPT_IMAGE = "RUNWAY_ALEPH2_PROMPT_IMAGE" + + +# Keyframe timing modes (anchored to the INPUT video). Stored on the chain item and used to +# choose the request model below. The values match the Aleph2 keyframe union field names. +KEYFRAME_MODE_SECONDS = "seconds" # absolute time, in seconds, from the start of the input video +KEYFRAME_MODE_AT = "at" # fraction [0.0, 1.0] of the input video duration + +# Prompt-image position modes (anchored to the OUTPUT video). Values match the Aleph2 position `type`. +PROMPT_IMAGE_MODE_TIMESTAMP = "timestamp" # absolute time, in seconds, from the start of the output video +PROMPT_IMAGE_MODE_POSITION = "position" # fraction [0.0, 1.0] of the output video duration + + +class RunwayAleph2KeyframeItem: + """A guidance image anchored to a point of the INPUT video (one Aleph2 ``keyframe``).""" + + def __init__(self, image, mode: str, value: float): + self.image = image + self.mode = mode # KEYFRAME_MODE_SECONDS | KEYFRAME_MODE_AT + self.value = value + + +class RunwayAleph2KeyframeChain: + """An ordered collection of keyframes, built by chaining Runway Aleph2 Keyframe nodes.""" + + def __init__(self): + self.items: list[RunwayAleph2KeyframeItem] = [] + + def add(self, item: RunwayAleph2KeyframeItem) -> None: + self.items.append(item) + + def clone(self) -> "RunwayAleph2KeyframeChain": + c = RunwayAleph2KeyframeChain() + c.items = list(self.items) + return c + + +class RunwayAleph2PromptImageItem: + """A guidance image anchored to a point of the OUTPUT video (one Aleph2 ``promptImage``).""" + + def __init__(self, image, mode: str, value: float): + self.image = image + self.mode = mode # PROMPT_IMAGE_MODE_TIMESTAMP | PROMPT_IMAGE_MODE_POSITION + self.value = value + + +class RunwayAleph2PromptImageChain: + """An ordered collection of prompt images, built by chaining Runway Aleph2 Prompt Image nodes.""" + + def __init__(self): + self.items: list[RunwayAleph2PromptImageItem] = [] + + def add(self, item: RunwayAleph2PromptImageItem) -> None: + self.items.append(item) + + def clone(self) -> "RunwayAleph2PromptImageChain": + c = RunwayAleph2PromptImageChain() + c.items = list(self.items) + return c + + +class RunwayAleph2KeyframeSeconds(BaseModel): + seconds: float = Field( + ..., + description="Absolute timestamp in seconds from the start of the input video when this guidance image should apply.", + ge=0.0, + ) + uri: str = Field(...) + + +class RunwayAleph2KeyframeAt(BaseModel): + at: float = Field( + ..., + description="Position as a fraction [0.0, 1.0] of the input video duration.", + ge=0.0, + le=1.0, + ) + uri: str = Field(...) + + +class RunwayAleph2TimestampPosition(BaseModel): + type: str = Field(default="timestamp") + timestampSeconds: float = Field( + ..., + description="Absolute timestamp in seconds from the start of the output video.", + ge=0.0, + ) + + +class RunwayAleph2RelativePosition(BaseModel): + type: str = Field(default="position") + positionPercentage: float = Field( + ..., + description="Position as a fraction [0.0, 1.0] of the total output video duration.", + ge=0.0, + le=1.0, + ) + + +class RunwayAleph2PromptImage(BaseModel): + position: RunwayAleph2TimestampPosition | RunwayAleph2RelativePosition + uri: str = Field(...) + + +class RunwayAleph2ContentModeration(BaseModel): + publicFigureThreshold: str = Field( + ..., + description='When set to "low", the content moderation system is less strict about ' + 'recognizable public figures. One of "auto" or "low".', + ) + + +class RunwayAleph2Request(BaseModel): + model: str = Field(default="aleph2") + promptText: str = Field( + ..., + description="A non-empty string describing what should appear in the output.", + min_length=1, + max_length=1000, + ) + videoUri: str = Field(...) + seed: int = Field(..., description="Random seed for generation", ge=0, le=4294967295) + contentModeration: RunwayAleph2ContentModeration = Field(...) + keyframes: list[RunwayAleph2KeyframeSeconds | RunwayAleph2KeyframeAt] | None = Field( + None, + description="Timed guidance images placed at specific points in the input video. Up to 5.", + ) + promptImage: list[RunwayAleph2PromptImage] | None = Field( + None, + description="Up to 5 image keyframes for guiding the edit at specific points in the output video.", + ) + + +class RunwayAleph2Response(BaseModel): + id: str | None = Field(None, description="Task ID") diff --git a/comfy_api_nodes/apis/stability.py b/comfy_api_nodes/apis/stability.py deleted file mode 100644 index 5b9b5ac7d..000000000 --- a/comfy_api_nodes/apis/stability.py +++ /dev/null @@ -1,147 +0,0 @@ -from enum import Enum -from typing import Optional - -from pydantic import BaseModel, Field, confloat - - -class StabilityFormat(str, Enum): - png = 'png' - jpeg = 'jpeg' - webp = 'webp' - - -class StabilityAspectRatio(str, Enum): - ratio_1_1 = "1:1" - ratio_16_9 = "16:9" - ratio_9_16 = "9:16" - ratio_3_2 = "3:2" - ratio_2_3 = "2:3" - ratio_5_4 = "5:4" - ratio_4_5 = "4:5" - ratio_21_9 = "21:9" - ratio_9_21 = "9:21" - - -def get_stability_style_presets(include_none=True): - presets = [] - if include_none: - presets.append("None") - return presets + [x.value for x in StabilityStylePreset] - - -class StabilityStylePreset(str, Enum): - _3d_model = "3d-model" - analog_film = "analog-film" - anime = "anime" - cinematic = "cinematic" - comic_book = "comic-book" - digital_art = "digital-art" - enhance = "enhance" - fantasy_art = "fantasy-art" - isometric = "isometric" - line_art = "line-art" - low_poly = "low-poly" - modeling_compound = "modeling-compound" - neon_punk = "neon-punk" - origami = "origami" - photographic = "photographic" - pixel_art = "pixel-art" - tile_texture = "tile-texture" - - -class Stability_SD3_5_Model(str, Enum): - sd3_5_large = "sd3.5-large" - # sd3_5_large_turbo = "sd3.5-large-turbo" - sd3_5_medium = "sd3.5-medium" - - -class Stability_SD3_5_GenerationMode(str, Enum): - text_to_image = "text-to-image" - image_to_image = "image-to-image" - - -class StabilityStable3_5Request(BaseModel): - model: str = Field(...) - mode: str = Field(...) - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - aspect_ratio: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - style_preset: Optional[str] = Field(None) - cfg_scale: float = Field(...) - strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) - - -class StabilityUpscaleConservativeRequest(BaseModel): - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - creativity: Optional[confloat(ge=0.2, le=0.5)] = Field(None) - - -class StabilityUpscaleCreativeRequest(BaseModel): - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - creativity: Optional[confloat(ge=0.1, le=0.5)] = Field(None) - style_preset: Optional[str] = Field(None) - - -class StabilityStableUltraRequest(BaseModel): - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - aspect_ratio: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - style_preset: Optional[str] = Field(None) - strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) - - -class StabilityStableUltraResponse(BaseModel): - image: Optional[str] = Field(None) - finish_reason: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - - -class StabilityResultsGetResponse(BaseModel): - image: Optional[str] = Field(None) - finish_reason: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - id: Optional[str] = Field(None) - name: Optional[str] = Field(None) - errors: Optional[list[str]] = Field(None) - status: Optional[str] = Field(None) - result: Optional[str] = Field(None) - - -class StabilityAsyncResponse(BaseModel): - id: Optional[str] = Field(None) - - -class StabilityTextToAudioRequest(BaseModel): - model: str = Field(...) - prompt: str = Field(...) - duration: int = Field(190, ge=1, le=190) - seed: int = Field(0, ge=0, le=4294967294) - steps: int = Field(8, ge=4, le=8) - output_format: str = Field("wav") - - -class StabilityAudioToAudioRequest(StabilityTextToAudioRequest): - strength: float = Field(0.01, ge=0.01, le=1.0) - - -class StabilityAudioInpaintRequest(StabilityTextToAudioRequest): - mask_start: int = Field(30, ge=0, le=190) - mask_end: int = Field(190, ge=0, le=190) - - -class StabilityAudioResponse(BaseModel): - audio: Optional[str] = Field(None) diff --git a/comfy_api_nodes/apis/sync_so.py b/comfy_api_nodes/apis/sync_so.py new file mode 100644 index 000000000..af9419580 --- /dev/null +++ b/comfy_api_nodes/apis/sync_so.py @@ -0,0 +1,49 @@ +from pydantic import BaseModel, Field + + +class SyncInputItem(BaseModel): + type: str = Field(..., description="Input kind: 'video', 'image' or 'audio'.") + url: str = Field(...) + + +class SyncActiveSpeakerDetection(BaseModel): + auto_detect: bool | None = Field( + None, description="Detect the active speaker automatically. Video input only; rejected for images." + ) + frame_number: int | None = Field( + None, description="Frame used for manual speaker selection. Must be 0 for image inputs." + ) + coordinates: list[int] | None = Field( + None, description="Pixel [x, y] of the speaker's face in the frame selected by frame_number." + ) + + +class SyncGenerationOptions(BaseModel): + sync_mode: str | None = Field( + None, + description="How to resolve an audio/video duration mismatch: " + "cut_off, bounce, loop, silence or remap. Ignored for image inputs.", + ) + i2v_prompt: str | None = Field( + None, description="Motion prompt for image-to-video generation. Image input only." + ) + active_speaker_detection: SyncActiveSpeakerDetection | None = Field(None) + + +class SyncGenerationRequest(BaseModel): + model: str = Field(..., description="Generation model, e.g. 'sync-3'.") + input: list[SyncInputItem] = Field( + ..., description="Exactly one visual input (video or image) plus one audio input." + ) + options: SyncGenerationOptions | None = Field(None) + + +class SyncGeneration(BaseModel): + """Subset of the Generation object returned by POST /v2/generate and GET /v2/generate/{id}.""" + + id: str = Field(...) + status: str = Field(..., description="PENDING | PROCESSING | COMPLETED | FAILED | REJECTED") + outputUrl: str | None = Field(None) + outputDuration: float | None = Field(None) + error: str | None = Field(None, description="Human-readable failure message.") + errorCode: str | None = Field(None, description="Stable machine-readable code from the GET /v2/errors catalog.") diff --git a/comfy_api_nodes/apis/tripo.py b/comfy_api_nodes/apis/tripo.py index 7ac81d42c..79913997a 100644 --- a/comfy_api_nodes/apis/tripo.py +++ b/comfy_api_nodes/apis/tripo.py @@ -208,6 +208,10 @@ class TripoMultiviewToModelRequest(BaseModel): quad: bool | None = Field(False, description="Whether to apply quad to the generated model") +class TripoTexturePrompt(BaseModel): + text: str | None = Field(None, description="Text guidance for texture generation") + + class TripoTextureModelRequest(BaseModel): type: TripoTaskType = Field(TripoTaskType.TEXTURE_MODEL, description="Type of task") original_model_task_id: str = Field(..., description="The task ID of the original model") @@ -219,6 +223,11 @@ class TripoTextureModelRequest(BaseModel): texture_alignment: TripoTextureAlignment | None = Field( TripoTextureAlignment.ORIGINAL_IMAGE, description="The texture alignment method" ) + texture_prompt: TripoTexturePrompt | None = Field( + None, + description="Optional guidance for texturing. Required in practice for imported models, " + "which carry no source image to infer texture from.", + ) class TripoRefineModelRequest(BaseModel): @@ -307,6 +316,17 @@ class TripoP1MultiviewToModelRequest(TripoP1CommonRequest): orientation: str | None = None +class TripoImportModelRequest(BaseModel): + """Request for the comfy-api composite import endpoint (/proxy/tripo/v2/openapi/import). + + The model file is uploaded to ComfyUI API storage first; the backend downloads it from + `url`, re-uploads it to Tripo's storage and creates the import_model task server-side. + """ + + url: str = Field(..., description="ComfyUI API storage download URL of the model file") + format: str = Field(..., description='File format: "glb", "fbx", "obj" or "stl"') + + class TripoTaskOutput(BaseModel): model: str | None = Field(None, description="URL to the model") base_model: str | None = Field(None, description="URL to the base model") diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index 79961ff9d..259c54ef9 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -534,6 +534,15 @@ class FluxEraseNode(IO.ComfyNode): max=25, tooltip="Expands the mask boundaries to ensure clean coverage of the object's edges.", ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="The random seed used for creating the noise.", + optional=True, + ), ], outputs=[IO.Image.Output()], hidden=[ @@ -553,6 +562,7 @@ class FluxEraseNode(IO.ComfyNode): image: Input.Image, mask: Input.Image, dilate_pixels: int = 10, + seed: int = 0, ) -> IO.NodeOutput: validate_image_dimensions(image, min_width=256, min_height=256) mask = resize_mask_to_image(mask, image) @@ -565,6 +575,7 @@ class FluxEraseNode(IO.ComfyNode): image=tensor_to_base64_string(image[:, :, :, :3]), # make sure image will have alpha channel removed mask=mask, dilate_pixels=dilate_pixels, + seed=seed, ), ) diff --git a/comfy_api_nodes/nodes_bria.py b/comfy_api_nodes/nodes_bria.py index 69b0233af..090154afb 100644 --- a/comfy_api_nodes/nodes_bria.py +++ b/comfy_api_nodes/nodes_bria.py @@ -1,14 +1,19 @@ +import av +import torch +from av.codec import CodecContext from typing_extensions import override from comfy_api.latest import IO, ComfyExtension, Input from comfy_api_nodes.apis.bria import ( BriaEditImageRequest, + BriaImageEditResponse, BriaRemoveBackgroundRequest, BriaRemoveBackgroundResponse, BriaRemoveVideoBackgroundRequest, BriaRemoveVideoBackgroundResponse, - BriaImageEditResponse, BriaStatusResponse, + BriaVideoGreenScreenRequest, + BriaVideoReplaceBackgroundRequest, InputModerationSettings, ) from comfy_api_nodes.util import ( @@ -284,7 +289,7 @@ class BriaRemoveVideoBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.14,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", ), ) @@ -316,6 +321,251 @@ class BriaRemoveVideoBackground(IO.ComfyNode): return IO.NodeOutput(await download_url_to_video_output(response.result.video_url)) +class BriaVideoGreenScreen(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="BriaVideoGreenScreen", + display_name="Bria Video Green Screen", + category="partner/video/Bria", + description="Replace a video's background with a solid chroma-key screen using Bria.", + inputs=[ + IO.Video.Input("video"), + IO.Combo.Input( + "green_shade", + options=["broadcast_green", "chroma_green", "blue_screen"], + tooltip="Solid chroma-key shade applied behind the foreground: " + "broadcast_green (#00B140), chroma_green (#00FF00), or blue_screen (#0000FF).", + ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=2147483647, + display_mode=IO.NumberDisplay.number, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[IO.Video.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + video: Input.Video, + green_shade: str, + seed: int, + ) -> IO.NodeOutput: + validate_video_duration(video, max_duration=60.0) + response = await sync_op( + cls, + ApiEndpoint(path="/proxy/bria/v2/video/edit/green_screen", method="POST"), + data=BriaVideoGreenScreenRequest( + video=await upload_video_to_comfyapi(cls, video), + green_shade=green_shade, + output_container_and_codec="mp4_h264", + seed=seed, + ), + response_model=BriaStatusResponse, + ) + response = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/bria/v2/status/{response.request_id}"), + status_extractor=lambda r: r.status, + response_model=BriaRemoveVideoBackgroundResponse, + ) + return IO.NodeOutput(await download_url_to_video_output(response.result.video_url)) + + +class BriaVideoReplaceBackground(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="BriaVideoReplaceBackground", + display_name="Bria Video Replace Background", + category="partner/video/Bria", + description="Replace a video's background with a supplied image or video using Bria. " + "The output keeps the foreground's resolution and frame rate; a background with a " + "different aspect ratio is stretched to fit, so match it for undistorted results.", + inputs=[ + IO.Video.Input("video", tooltip="Foreground video whose background is replaced."), + IO.Image.Input( + "background_image", + optional=True, + tooltip="Background image to composite behind the foreground. " + "Provide either a background image or a background video, not both.", + ), + IO.Video.Input( + "background_video", + optional=True, + tooltip="Background video to composite behind the foreground. " + "Provide either a background image or a background video, not both.", + ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=2147483647, + display_mode=IO.NumberDisplay.number, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[IO.Video.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + video: Input.Video, + seed: int, + background_image: Input.Image | None = None, + background_video: Input.Video | None = None, + ) -> IO.NodeOutput: + if (background_image is None) == (background_video is None): + raise ValueError("Provide either a background image or a background video, not both.") + validate_video_duration(video, max_duration=60.0) + if background_video is not None: + validate_video_duration(background_video, max_duration=60.0) + background_url = await upload_video_to_comfyapi(cls, background_video, wait_label="Uploading background") + else: + # Bria's replace_background 500s on RGBA, so drop the alpha channel before upload. + background_url = await upload_image_to_comfyapi( + cls, background_image[:, :, :, :3], wait_label="Uploading background" + ) + response = await sync_op( + cls, + ApiEndpoint(path="/proxy/bria/v2/video/edit/replace_background", method="POST"), + data=BriaVideoReplaceBackgroundRequest( + video=await upload_video_to_comfyapi(cls, video), + background_url=background_url, + output_container_and_codec="mp4_h264", + seed=seed, + ), + response_model=BriaStatusResponse, + ) + response = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/bria/v2/status/{response.request_id}"), + status_extractor=lambda r: r.status, + response_model=BriaRemoveVideoBackgroundResponse, + ) + return IO.NodeOutput(await download_url_to_video_output(response.result.video_url)) + + +def _video_to_images_and_mask(video: Input.Video) -> tuple[Input.Image, Input.Mask]: + """Decode a transparent webm (VP9 + alpha) into image frames and an alpha mask. + + VP9 keeps its alpha in a side layer that PyAV's default vp9 decoder drops, so the frames + are decoded with libvpx-vp9. Returns RGB images [B,H,W,3] in 0..1 and a mask [B,H,W] + following the Load Image convention (1 = transparent) for compositing or Save WEBM. + """ + rgb_frames: list[torch.Tensor] = [] + alpha_frames: list[torch.Tensor] = [] + with av.open(video.get_stream_source(), mode="r") as container: + stream = container.streams.video[0] + decoder = CodecContext.create("libvpx-vp9", "r") if stream.codec_context.name == "vp9" else None + for packet in container.demux(stream): + for frame in (decoder.decode(packet) if decoder is not None else packet.decode()): + rgba = torch.from_numpy(frame.to_ndarray(format="rgba")).float() / 255.0 + rgb_frames.append(rgba[..., :3]) + alpha_frames.append(rgba[..., 3]) + images = torch.stack(rgb_frames) if rgb_frames else torch.zeros(0, 0, 0, 3) + mask = (1.0 - torch.stack(alpha_frames)) if alpha_frames else torch.zeros((images.shape[0], 64, 64)) + return images, mask + + +class BriaTransparentVideoBackground(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="BriaTransparentVideoBackground", + display_name="Bria Remove Video Background (Transparent)", + category="partner/video/Bria", + description="Remove the background from a video using Bria and return the cut-out frames " + "plus an alpha mask. Connect both to a compositing node, or feed them to Save WEBM to " + "write a transparent video.", + inputs=[ + IO.Video.Input("video"), + IO.Int.Input( + "seed", + default=0, + min=0, + max=2147483647, + display_mode=IO.NumberDisplay.number, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[ + IO.Image.Output(display_name="images"), + IO.Mask.Output(display_name="mask"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + video: Input.Video, + seed: int, + ) -> IO.NodeOutput: + validate_video_duration(video, max_duration=60.0) + response = await sync_op( + cls, + ApiEndpoint(path="/proxy/bria/v2/video/edit/remove_background", method="POST"), + data=BriaRemoveVideoBackgroundRequest( + video=await upload_video_to_comfyapi(cls, video), + background_color="Transparent", + output_container_and_codec="webm_vp9", + seed=seed, + ), + response_model=BriaStatusResponse, + ) + response = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/bria/v2/status/{response.request_id}"), + status_extractor=lambda r: r.status, + response_model=BriaRemoveVideoBackgroundResponse, + ) + video_out = await download_url_to_video_output(response.result.video_url) + images, mask = _video_to_images_and_mask(video_out) + return IO.NodeOutput(images, mask) + + class BriaExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -323,6 +573,9 @@ class BriaExtension(ComfyExtension): BriaImageEditNode, BriaRemoveImageBackground, BriaRemoveVideoBackground, + BriaVideoGreenScreen, + BriaVideoReplaceBackground, + BriaTransparentVideoBackground, ] diff --git a/comfy_api_nodes/nodes_bytedance.py b/comfy_api_nodes/nodes_bytedance.py index d8885a7e5..a84399ad3 100644 --- a/comfy_api_nodes/nodes_bytedance.py +++ b/comfy_api_nodes/nodes_bytedance.py @@ -1,3 +1,4 @@ +import base64 import hashlib import logging import math @@ -7,6 +8,7 @@ from io import BytesIO import torch from typing_extensions import override +from comfy.utils import common_upscale from comfy_api.latest import IO, ComfyExtension, Input, Types from comfy_api_nodes.apis.bytedance import ( RECOMMENDED_PRESETS, @@ -14,12 +16,16 @@ from comfy_api_nodes.apis.bytedance import ( RECOMMENDED_PRESETS_SEEDREAM_4_0, RECOMMENDED_PRESETS_SEEDREAM_4_5, RECOMMENDED_PRESETS_SEEDREAM_5_LITE, - SEEDANCE2_PRICE_PER_1K_TOKENS, + RECOMMENDED_PRESETS_SEEDREAM_5_PRO, SEEDANCE2_REF_VIDEO_PIXEL_LIMITS, VIDEO_TASKS_EXECUTION_TIME, GetAssetResponse, Image2VideoTaskCreationRequest, ImageTaskCreationResponse, + SeedAudioConfig, + SeedAudioReference, + SeedAudioRequest, + SeedAudioResponse, Seedance2TaskCreationRequest, SeedanceCreateAssetRequest, SeedanceCreateAssetResponse, @@ -28,6 +34,7 @@ from comfy_api_nodes.apis.bytedance import ( SeedanceVirtualLibraryCreateAssetRequest, Seedream4Options, Seedream4TaskCreationRequest, + Seedream5OptimizePromptOptions, TaskAudioContent, TaskAudioContentUrl, TaskCreationResponse, @@ -39,9 +46,12 @@ from comfy_api_nodes.apis.bytedance import ( TaskVideoContentUrl, Text2ImageTaskCreationRequest, Text2VideoTaskCreationRequest, + seedance2_price_per_1k_tokens, ) from comfy_api_nodes.util import ( ApiEndpoint, + audio_bytes_to_audio_input, + audio_input_to_mp3, download_url_to_image_tensor, download_url_to_video_output, downscale_image_tensor_by_max_side, @@ -50,11 +60,14 @@ from comfy_api_nodes.util import ( image_tensor_pair_to_batch, poll_op, sync_op, + tensor_to_base64_string, upload_audio_to_comfyapi, upload_image_to_comfyapi, upload_images_to_comfyapi, upload_video_to_comfyapi, + upscale_image_tensor_to_min_pixels, upscale_video_to_min_pixels, + validate_audio_duration, validate_image_aspect_ratio, validate_image_dimensions, validate_string, @@ -69,12 +82,14 @@ _VERIFICATION_POLL_TIMEOUT_SEC = 120 _VERIFICATION_POLL_INTERVAL_SEC = 3 SEEDREAM_MODELS = { + "seedream 5.0 pro": "seedream-5-0-pro-260628", "seedream 5.0 lite": "seedream-5-0-260128", "seedream-4-5-251128": "seedream-4-5-251128", "seedream-4-0-250828": "seedream-4-0-250828", } SEEDREAM_PRESETS = { + "seedream-5-0-pro-260628": RECOMMENDED_PRESETS_SEEDREAM_5_PRO, "seedream-5-0-260128": RECOMMENDED_PRESETS_SEEDREAM_5_LITE, "seedream-4-5-251128": RECOMMENDED_PRESETS_SEEDREAM_4_5, "seedream-4-0-250828": RECOMMENDED_PRESETS_SEEDREAM_4_0, @@ -88,6 +103,7 @@ BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT = "/proxy/byteplus-seedance2/api/v3/cont SEEDANCE_MODELS = { "Seedance 2.0": "dreamina-seedance-2-0-260128", "Seedance 2.0 Fast": "dreamina-seedance-2-0-fast-260128", + "Seedance 2.0 Mini": "dreamina-seedance-2-0-mini", } DEPRECATED_MODELS = {"seedance-1-0-lite-t2v-250428", "seedance-1-0-lite-i2v-250428"} @@ -131,6 +147,44 @@ def _prepare_seedance_image(image: Input.Image) -> Input.Image: return image +# Supported output aspect ratios, used to pre-size FLF frames to matching pixel pair to avoid the 1080p stretch jump. +SEEDANCE2_RATIO_WH = { + "16:9": (16, 9), + "4:3": (4, 3), + "1:1": (1, 1), + "3:4": (3, 4), + "9:16": (9, 16), + "21:9": (21, 9), +} +SEEDANCE2_RES_SHORT_SIDE = {"480p": 480, "720p": 720, "1080p": 1080, "4k": 2160} + + +def _seedance2_target_dims(resolution: str, ratio: str, image: torch.Tensor) -> tuple[int, int]: + """Exact supported output (width, height) for (resolution, ratio). + + The shorter side equals the resolution number (e.g. 1080p 16:9 -> 1920x1080). For ratio + "adaptive" (or any unexpected value) the ratio is derived from the image's own aspect, snapped + to the nearest supported ratio, so the output keeps the frame's orientation. + """ + short = SEEDANCE2_RES_SHORT_SIDE[resolution] + if ratio not in SEEDANCE2_RATIO_WH: + aspect = image.shape[-2] / image.shape[-3] # W / H; tensor is (B, H, W, C) + ratio = min(SEEDANCE2_RATIO_WH, key=lambda k: abs(SEEDANCE2_RATIO_WH[k][0] / SEEDANCE2_RATIO_WH[k][1] - aspect)) + rw, rh = SEEDANCE2_RATIO_WH[ratio] + if rw >= rh: # landscape or square: shorter side is the height + out_w, out_h = round(short * rw / rh), short + else: # portrait: shorter side is the width + out_w, out_h = short, round(short * rh / rw) + return out_w - out_w % 2, out_h - out_h % 2 + + +def _resize_to_exact(image: torch.Tensor, width: int, height: int) -> torch.Tensor: + """Center-crop to the target aspect and resize to exactly width x height (lanczos).""" + samples = image.movedim(-1, 1) # (B, H, W, C) -> (B, C, H, W) + resized = common_upscale(samples, width, height, "lanczos", "center") + return resized.movedim(1, -1) + + async def _resolve_reference_assets( cls: type[IO.ComfyNode], asset_ids: list[str], @@ -338,9 +392,9 @@ async def _seedance_virtual_library_upload_video_asset( return f"asset://{create_resp.asset_id}" -def _seedance2_price_extractor(model_id: str, has_video_input: bool): +def _seedance2_price_extractor(model_id: str, has_video_input: bool, resolution: str): """Returns a price_extractor closure for Seedance 2.0 poll_op.""" - rate = SEEDANCE2_PRICE_PER_1K_TOKENS.get((model_id, has_video_input)) + rate = seedance2_price_per_1k_tokens(model_id, has_video_input, resolution) if rate is None: return None @@ -693,8 +747,15 @@ class ByteDanceSeedreamNode(IO.ComfyNode): return IO.NodeOutput(torch.cat([await download_url_to_image_tensor(i) for i in urls])) -def _seedream_model_inputs(*, max_ref_images: int, presets: list): - return [ +def _seedream_model_inputs( + *, + max_ref_images: int, + presets: list, + max_width: int = 6240, + max_height: int = 4992, + supports_batch: bool = True, +): + inputs = [ IO.Combo.Input( "size_preset", options=[label for label, _, _ in presets], @@ -704,7 +765,7 @@ def _seedream_model_inputs(*, max_ref_images: int, presets: list): "width", default=2048, min=1024, - max=6240, + max=max_width, step=2, tooltip="Custom width for image. Value is working only if `size_preset` is set to `Custom`", ), @@ -712,22 +773,27 @@ def _seedream_model_inputs(*, max_ref_images: int, presets: list): "height", default=2048, min=1024, - max=4992, + max=max_height, step=2, tooltip="Custom height for image. Value is working only if `size_preset` is set to `Custom`", ), - IO.Int.Input( - "max_images", - default=1, - min=1, - max=max_ref_images, - step=1, - display_mode=IO.NumberDisplay.number, - tooltip="Maximum number of images to generate. With 1, exactly one image is produced. " - "With >1, the model generates between 1 and max_images related images " - "(e.g., story scenes, character variations). " - "Total images (input + generated) cannot exceed 15.", - ), + ] + if supports_batch: + inputs.append( + IO.Int.Input( + "max_images", + default=1, + min=1, + max=max_ref_images, + step=1, + display_mode=IO.NumberDisplay.number, + tooltip="Maximum number of images to generate. With 1, exactly one image is produced. " + "With >1, the model generates between 1 and max_images related images " + "(e.g., story scenes, character variations). " + "Total images (input + generated) cannot exceed 15.", + ) + ) + inputs.append( IO.Autogrow.Input( "images", template=IO.Autogrow.TemplateNames( @@ -737,14 +803,18 @@ def _seedream_model_inputs(*, max_ref_images: int, presets: list): ), tooltip=f"Optional reference image(s) for image-to-image or multi-reference generation. " f"Up to {max_ref_images} images.", - ), - IO.Boolean.Input( - "fail_on_partial", - default=False, - tooltip="If enabled, abort execution if any requested images are missing or return an error.", - advanced=True, - ), - ] + ) + ) + if supports_batch: + inputs.append( + IO.Boolean.Input( + "fail_on_partial", + default=False, + tooltip="If enabled, abort execution if any requested images are missing or return an error.", + advanced=True, + ) + ) + return inputs class ByteDanceSeedreamNodeV2(IO.ComfyNode): @@ -766,6 +836,16 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "seedream 5.0 pro", + _seedream_model_inputs( + max_ref_images=10, + presets=RECOMMENDED_PRESETS_SEEDREAM_5_PRO, + max_width=3136, + max_height=2496, + supports_batch=False, + ), + ), IO.DynamicCombo.Option( "seedream 5.0 lite", _seedream_model_inputs(max_ref_images=14, presets=RECOMMENDED_PRESETS_SEEDREAM_5_LITE), @@ -796,6 +876,17 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): tooltip='Whether to add an "AI generated" watermark to the image.', advanced=True, ), + IO.Boolean.Input( + "thinking", + default=True, + tooltip=( + "Enable the model's prompt-optimization reasoning ('thinking') for better adherence. " + "Can substantially increase generation time — notably on Seedream 5.0 Pro. " + "Can only be disabled for text-to-image (not when reference images are provided)." + ), + optional=True, + advanced=True, + ), ], outputs=[ IO.Image.Output(), @@ -807,15 +898,27 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model"]), + depends_on=IO.PriceBadgeDepends( + widgets=["model", "model.size_preset", "model.width", "model.height"] + ), expr=""" ( - $price := $contains(widgets.model, "5.0 lite") ? 0.035 : - $contains(widgets.model, "4-5") ? 0.04 : 0.03; + $sp := $lookup(widgets, "model.size_preset"); + $px := $lookup(widgets, "model.width") * $lookup(widgets, "model.height"); + $isPro := $contains(widgets.model, "5.0 pro"); + $price := $isPro + ? ( + $contains($sp, "custom") + ? ($px <= 2360000 ? 0.045 : 0.09) + : ($contains($sp, "1k") ? 0.045 : 0.09) + ) + : $contains(widgets.model, "5.0 lite") ? 0.035 + : $contains(widgets.model, "4-5") ? 0.04 + : 0.03; { - "type":"usd", + "type": "usd", "usd": $price, - "format": { "suffix":" x images/Run", "approximate": true } + "format": { "suffix": $isPro ? "/Image" : " x images/Run", "approximate": true } } ) """, @@ -829,10 +932,12 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): model: dict, seed: int = 0, watermark: bool = False, + thinking: bool = True, ) -> IO.NodeOutput: validate_string(prompt, strip_whitespace=True, min_length=1) model_id = SEEDREAM_MODELS[model["model"]] presets = SEEDREAM_PRESETS[model_id] + is_pro = "seedream-5-0-pro" in model_id size_preset = model.get("size_preset", presets[0][0]) width = model.get("width", 2048) @@ -852,19 +957,29 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): out_num_pixels = w * h mp_provided = out_num_pixels / 1_000_000.0 - if ("seedream-4-5" in model_id or "seedream-5-0" in model_id) and out_num_pixels < 3686400: - raise ValueError( - f"Minimum image resolution for the selected model is 3.68MP, but {mp_provided:.2f}MP provided." - ) - if "seedream-4-0" in model_id and out_num_pixels < 921600: - raise ValueError( - f"Minimum image resolution that the selected model can generate is 0.92MP, " - f"but {mp_provided:.2f}MP provided." - ) - if out_num_pixels > 16_777_216: - raise ValueError( - f"Maximum image resolution for the selected model is 16.78MP, but {mp_provided:.2f}MP provided." - ) + if is_pro: + if out_num_pixels < 921_600: + raise ValueError( + f"Minimum image resolution for the selected model is 0.92MP, but {mp_provided:.2f}MP provided." + ) + if out_num_pixels > 4_194_304: + raise ValueError( + f"Maximum image resolution for the selected model is 4.19MP, but {mp_provided:.2f}MP provided." + ) + else: + if ("seedream-4-5" in model_id or "seedream-5-0" in model_id) and out_num_pixels < 3_686_400: + raise ValueError( + f"Minimum image resolution for the selected model is 3.68MP, but {mp_provided:.2f}MP provided." + ) + if "seedream-4-0" in model_id and out_num_pixels < 921_600: + raise ValueError( + f"Minimum image resolution that the selected model can generate is 0.92MP, " + f"but {mp_provided:.2f}MP provided." + ) + if out_num_pixels > 16_777_216: + raise ValueError( + f"Maximum image resolution for the selected model is 16.78MP, but {mp_provided:.2f}MP provided." + ) image_tensors: list[Input.Image] = [t for t in images_dict.values() if t is not None] n_input_images = sum(get_number_of_images(t) for t in image_tensors) @@ -877,6 +992,10 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): raise ValueError( "The maximum number of generated images plus the number of reference images cannot exceed 15." ) + if not thinking and n_input_images > 0: + raise ValueError( + "'thinking' can only be disabled for text-to-image; enable it when using reference images." + ) reference_images_urls: list[str] = [] if image_tensors: @@ -890,6 +1009,9 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): wait_label="Uploading reference images", ) + optimize_prompt_options = None + if n_input_images == 0: + optimize_prompt_options = Seedream5OptimizePromptOptions(thinking="enabled" if thinking else "disabled") response = await sync_op( cls, ApiEndpoint(path=BYTEPLUS_IMAGE_ENDPOINT, method="POST"), @@ -900,9 +1022,10 @@ class ByteDanceSeedreamNodeV2(IO.ComfyNode): image=reference_images_urls, size=f"{w}x{h}", seed=seed, - sequential_image_generation=sequential_image_generation, - sequential_image_generation_options=Seedream4Options(max_images=max_images), + sequential_image_generation=None if is_pro else sequential_image_generation, + sequential_image_generation_options=None if is_pro else Seedream4Options(max_images=max_images), watermark=watermark, + optimize_prompt_options=optimize_prompt_options, ), ) if len(response.data) == 1: @@ -1582,10 +1705,12 @@ class ByteDance2TextToVideoNode(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ - IO.DynamicCombo.Option("Seedance 2.0", _seedance2_text_inputs(["480p", "720p", "1080p"])), + IO.DynamicCombo.Option("Seedance 2.0", _seedance2_text_inputs(["480p", "720p", "1080p", "4k"])), IO.DynamicCombo.Option("Seedance 2.0 Fast", _seedance2_text_inputs(["480p", "720p"])), + IO.DynamicCombo.Option("Seedance 2.0 Mini", _seedance2_text_inputs(["480p", "720p"])), ], - tooltip="Seedance 2.0 for maximum quality; Seedance 2.0 Fast for speed optimization.", + tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; " + "Mini for the fastest, lowest-cost generation.", ), IO.Int.Input( "seed", @@ -1621,11 +1746,16 @@ class ByteDance2TextToVideoNode(IO.ComfyNode): $rate480 := 10044; $rate720 := 21600; $rate1080 := 48800; + $rate4k := 195200; $m := widgets.model; - $pricePer1K := $contains($m, "fast") ? 0.008008 : 0.01001; $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $rate := $res = "1080p" ? $rate1080 : + $pricePer1K := $res = "4k" ? 0.00572 : + $res = "1080p" ? 0.011011 : + $contains($m, "mini") ? 0.005005 : + $contains($m, "fast") ? 0.008008 : 0.01001; + $rate := $res = "4k" ? $rate4k : + $res = "1080p" ? $rate1080 : $res = "720p" ? $rate720 : $rate480; $cost := $dur * $rate * $pricePer1K / 1000; @@ -1664,7 +1794,7 @@ class ByteDance2TextToVideoNode(IO.ComfyNode): ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"), response_model=TaskStatusResponse, status_extractor=lambda r: r.status, - price_extractor=_seedance2_price_extractor(model_id, has_video_input=False), + price_extractor=_seedance2_price_extractor(model_id, has_video_input=False, resolution=model["resolution"]), poll_interval=9, ) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) @@ -1685,14 +1815,19 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): options=[ IO.DynamicCombo.Option( "Seedance 2.0", - _seedance2_text_inputs(["480p", "720p", "1080p"], default_ratio="adaptive"), + _seedance2_text_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"), ), IO.DynamicCombo.Option( "Seedance 2.0 Fast", _seedance2_text_inputs(["480p", "720p"], default_ratio="adaptive"), ), + IO.DynamicCombo.Option( + "Seedance 2.0 Mini", + _seedance2_text_inputs(["480p", "720p"], default_ratio="adaptive"), + ), ], - tooltip="Seedance 2.0 for maximum quality; Seedance 2.0 Fast for speed optimization.", + tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; " + "Mini for the fastest, lowest-cost generation.", ), IO.Image.Input( "first_frame", @@ -1752,11 +1887,16 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): $rate480 := 10044; $rate720 := 21600; $rate1080 := 48800; + $rate4k := 195200; $m := widgets.model; - $pricePer1K := $contains($m, "fast") ? 0.008008 : 0.01001; $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $rate := $res = "1080p" ? $rate1080 : + $pricePer1K := $res = "4k" ? 0.00572 : + $res = "1080p" ? 0.011011 : + $contains($m, "mini") ? 0.005005 : + $contains($m, "fast") ? 0.008008 : 0.01001; + $rate := $res = "4k" ? $rate4k : + $res = "1080p" ? $rate1080 : $res = "720p" ? $rate720 : $rate480; $cost := $dur * $rate * $pricePer1K / 1000; @@ -1790,10 +1930,28 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): if last_frame is not None and last_frame_asset_id: raise ValueError("Provide only one of last_frame or last_frame_asset_id, not both.") - if first_frame is not None: - first_frame = _prepare_seedance_image(first_frame) - if last_frame is not None: - last_frame = _prepare_seedance_image(last_frame) + request_ratio = model["ratio"] + if first_frame_asset_id or last_frame_asset_id: + if first_frame is not None: + first_frame = _prepare_seedance_image(first_frame) + if last_frame is not None: + last_frame = _prepare_seedance_image(last_frame) + else: + # The 1080p FLF stretch fix (pre-size frames to a supported pixel pair + submit ratio="adaptive") + # only applies to local image inputs we can resize. + request_ratio = "adaptive" + target_dims: tuple[int, int] | None = None + if first_frame is not None: + validate_image_aspect_ratio(first_frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5 + validate_image_dimensions(first_frame, min_width=300, min_height=300) + target_dims = _seedance2_target_dims(model["resolution"], model["ratio"], first_frame) + first_frame = _resize_to_exact(first_frame, *target_dims) + if last_frame is not None: + validate_image_aspect_ratio(last_frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5 + validate_image_dimensions(last_frame, min_width=300, min_height=300) + if target_dims is None: + target_dims = _seedance2_target_dims(model["resolution"], model["ratio"], last_frame) + last_frame = _resize_to_exact(last_frame, *target_dims) asset_ids_to_resolve = [a for a in (first_frame_asset_id, last_frame_asset_id) if a] image_assets: dict[str, str] = {} @@ -1844,7 +2002,7 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): content=content, generate_audio=model["generate_audio"], resolution=model["resolution"], - ratio=model["ratio"], + ratio=request_ratio, duration=model["duration"], seed=seed, watermark=watermark, @@ -1856,7 +2014,7 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"), response_model=TaskStatusResponse, status_extractor=lambda r: r.status, - price_extractor=_seedance2_price_extractor(model_id, has_video_input=False), + price_extractor=_seedance2_price_extractor(model_id, has_video_input=False, resolution=model["resolution"]), poll_interval=9, ) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) @@ -1953,14 +2111,19 @@ class ByteDance2ReferenceNode(IO.ComfyNode): options=[ IO.DynamicCombo.Option( "Seedance 2.0", - _seedance2_reference_inputs(["480p", "720p", "1080p"], default_ratio="adaptive"), + _seedance2_reference_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"), ), IO.DynamicCombo.Option( "Seedance 2.0 Fast", _seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"), ), + IO.DynamicCombo.Option( + "Seedance 2.0 Mini", + _seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"), + ), ], - tooltip="Seedance 2.0 for maximum quality; Seedance 2.0 Fast for speed optimization.", + tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; " + "Mini for the fastest, lowest-cost generation.", ), IO.Int.Input( "seed", @@ -1999,13 +2162,21 @@ class ByteDance2ReferenceNode(IO.ComfyNode): $rate480 := 10044; $rate720 := 21600; $rate1080 := 48800; + $rate4k := 195200; $m := widgets.model; $hasVideo := $lookup(inputGroups, "model.reference_videos") > 0; - $noVideoPricePer1K := $contains($m, "fast") ? 0.008008 : 0.01001; - $videoPricePer1K := $contains($m, "fast") ? 0.004719 : 0.006149; $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $rate := $res = "1080p" ? $rate1080 : + $noVideoPricePer1K := $res = "4k" ? 0.00572 : + $res = "1080p" ? 0.011011 : + $contains($m, "mini") ? 0.005005 : + $contains($m, "fast") ? 0.008008 : 0.01001; + $videoPricePer1K := $res = "4k" ? 0.003432 : + $res = "1080p" ? 0.006721 : + $contains($m, "mini") ? 0.003003 : + $contains($m, "fast") ? 0.004719 : 0.006149; + $rate := $res = "4k" ? $rate4k : + $res = "1080p" ? $rate1080 : $res = "720p" ? $rate720 : $rate480; $noVideoCost := $dur * $rate * $noVideoPricePer1K / 1000; @@ -2201,7 +2372,9 @@ class ByteDance2ReferenceNode(IO.ComfyNode): ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"), response_model=TaskStatusResponse, status_extractor=lambda r: r.status, - price_extractor=_seedance2_price_extractor(model_id, has_video_input=has_video_input), + price_extractor=_seedance2_price_extractor( + model_id, has_video_input=has_video_input, resolution=model["resolution"] + ), poll_interval=9, ) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) @@ -2384,6 +2557,311 @@ class ByteDanceCreateVideoAsset(IO.ComfyNode): return IO.NodeOutput(asset_id, resolved_group) +MODE_TEXT = "text only" +MODE_AUDIO = "audio reference" +MODE_IMAGE = "image reference" +MODE_SPEAKER = "preset voice" + +# (speaker_id, display_label) for built-in TTS 2.0 voices; resolvable ids are account-scoped. +SEED_AUDIO_PRESET_VOICES: list[tuple[str, str]] = [ + ("zh_female_vv_uranus_bigtts", "Vivi (Female, multilingual)"), + ("zh_female_xiaohe_uranus_bigtts", "Mindy (Female, multilingual)"), + ("en_female_stokie_uranus_bigtts", "Stokie (Female, English)"), + ("en_female_dacey_uranus_bigtts", "Dacey (Female, English)"), + ("en_male_tim_uranus_bigtts", "Tim (Male, English)"), + ("zh_male_m191_uranus_bigtts", "Kian (Male, multilingual)"), + ("zh_male_taocheng_uranus_bigtts", "Cedric (Male, multilingual)"), + ("zh_male_sophie_uranus_bigtts", "Sophie (Female, multilingual)"), + ("zh_female_yingyujiaoxue_uranus_bigtts", "Jean (Female, multilingual)"), + ("zh_male_dayi_uranus_bigtts", "Magnus (Male, multilingual)"), + ("zh_female_mizai_uranus_bigtts", "Mabel (Female, multilingual)"), + ("zh_female_jitangnv_uranus_bigtts", "Nadia (Female, multilingual)"), + ("zh_female_meilinvyou_uranus_bigtts", "Opal (Female, multilingual)"), + ("zh_female_liuchangnv_uranus_bigtts", "Pearl (Female, multilingual)"), + ("zh_male_ruyayichen_uranus_bigtts", "Quentin (Male, multilingual)"), + ("zh_female_vivo_uranus_bigtts", "Vienna (Female, multilingual)"), + ("zh_female_xiaoai_uranus_bigtts", "Alina (Female, multilingual)"), + ("zh_female_cancan_uranus_bigtts", "Corinne (Female, multilingual)"), + ("zh_female_tianmeixiaoyuan_uranus_bigtts", "Esther (Female, multilingual)"), + ("zh_female_tianmeitaozi_uranus_bigtts", "Freya (Female, multilingual)"), + ("zh_female_shuangkuaisisi_uranus_bigtts", "Gigi (Female, multilingual)"), + ("zh_female_peiqi_uranus_bigtts", "Holly (Female, multilingual)"), + ("zh_female_xiaoxue_uranus_bigtts", "Lyla (Female, multilingual)"), + ("zh_female_yuanqi_uranus_bigtts", "Daisy (Female, multilingual)"), + ("zh_female_kefunvsheng_uranus_bigtts", "Tracy (Female, multilingual)"), + ("zh_male_shaonianzixin_uranus_bigtts", "Jess (Male, multilingual)"), + ("zh_female_linjianvhai_uranus_bigtts", "Pinky (Female, multilingual)"), + ("zh_female_kiwi_uranus_bigtts", "Sweety (Female, multilingual)"), + ("zh_female_sajiaoxuemei_uranus_bigtts", "Sandy (Female, multilingual)"), + ("de_male_seven_uranus_bigtts", "Sven (Male, German)"), + ("jp_female_minimi_uranus_bigtts", "Minimi (Female, Japanese)"), + ("fr_male_usseau_uranus_bigtts", "Usseau (Male, French)"), + ("es_male_felipe_uranus_bigtts", "Felipe (Male, Spanish)"), + ("id_male_han_uranus_bigtts", "Han (Male, Indonesian)"), + ("pt_male_martins_uranus_bigtts", "Martins (Male, Portuguese)"), + ("it_male_enzo_uranus_bigtts", "Enzo (Male, Italian)"), + ("kr_male_shane_uranus_bigtts", "Shane (Male, Korean)"), + ("zh_male_liufei_uranus_bigtts", "Felix (Male, Chinese)"), + ("zh_female_qingxinnvsheng_uranus_bigtts", "Celeste (Female, Chinese)"), + ("zh_male_sunwukong_uranus_bigtts", "Monkey King (Male, Chinese)"), +] +SEED_AUDIO_VOICE_OPTIONS = [label for _, label in SEED_AUDIO_PRESET_VOICES] +SEED_AUDIO_VOICE_MAP = {label: speaker_id for speaker_id, label in SEED_AUDIO_PRESET_VOICES} + +_AUDIO_TAG_RE = re.compile(r"@Audio(\d+)", re.IGNORECASE) + + +def max_audio_tag(prompt: str) -> int: + """Highest N referenced as @AudioN in the prompt (0 if none).""" + nums = [int(m) for m in _AUDIO_TAG_RE.findall(prompt or "")] + return max(nums) if nums else 0 + + +def connected_audio_indices(reference_mode: dict) -> list[int]: + """Indices (1-based) of connected reference_audio sockets, in order.""" + return [ + i + for i in range(1, 3 + 1) + if reference_mode.get(f"reference_audio_{i}") is not None + ] + + +def validate_seed_audio_inputs( + text_prompt: str, + mode: str, + audio_indices: list[int], + has_image: bool, + preset_voice: str | None = None, +) -> None: + validate_string(text_prompt, field_name="text_prompt", min_length=1, max_length=3000) + max_tag = max_audio_tag(text_prompt) + + if mode == MODE_TEXT: + if max_tag: + raise ValueError( + f"The prompt references @Audio{max_tag}, but reference mode is '{MODE_TEXT}'. " + f"Switch to '{MODE_AUDIO}' and connect the reference clip(s)." + ) + elif mode == MODE_AUDIO: + if not audio_indices: + raise ValueError( + f"Reference mode '{MODE_AUDIO}' requires at least one reference_audio input " + f"(or switch to '{MODE_TEXT}')." + ) + if audio_indices != list(range(1, len(audio_indices) + 1)): + raise ValueError( + "Connect reference_audio inputs in order without gaps: reference_audio_1, then _2, then _3." + ) + if max_tag > len(audio_indices): + raise ValueError( + f"The prompt references @Audio{max_tag}, but only {len(audio_indices)} " + f"reference audio(s) are connected." + ) + elif mode == MODE_IMAGE: + if not has_image: + raise ValueError(f"Reference mode '{MODE_IMAGE}' requires a reference_image input.") + if max_tag: + raise ValueError( + f"@AudioN tags are not used in '{MODE_IMAGE}' mode; the prompt should contain " + f"only the text to synthesize." + ) + elif mode == MODE_SPEAKER: + if not preset_voice or preset_voice not in SEED_AUDIO_VOICE_MAP: + raise ValueError(f"Reference mode '{MODE_SPEAKER}' requires selecting a preset voice.") + if max_tag > 1: + raise ValueError( + f"'{MODE_SPEAKER}' mode uses a single voice, so @Audio{max_tag} is out of range. " + f"Remove the @AudioN tags — the whole prompt is read in the selected voice." + ) + else: + raise ValueError(f"Unknown reference mode: {mode!r}") + + +class ByteDanceSeedAudioNode(IO.ComfyNode): + + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="ByteDanceSeedAudio", + display_name="ByteDance Seed Audio 1.0", + category="partner/audio/ByteDance", + description=( + "Generate speech, music, sound effects and multi-speaker dialogue from a single prompt " + "with ByteDance Seed Audio 1.0. Describe the voice(s), emotion, ambience, background music " + "and sound effects in the prompt, and include the lines to speak. Optionally pick a built-in " + "preset voice, clone voices from up to 3 reference clips (tagged @Audio1-3 in the prompt), " + "or derive a voice from a character image. Up to 2 minutes of audio per run." + ), + inputs=[ + IO.String.Input( + "text_prompt", + multiline=True, + default="", + tooltip=( + "Describe the voice(s), emotion, pacing, ambience, background music and sound " + "effects, and include the lines to speak (name characters inline for dialogue). " + "In 'audio reference' mode, refer to connected clips by order as @Audio1, @Audio2, " + "@Audio3. Maximum 3000 characters." + ), + ), + IO.DynamicCombo.Input( + "reference_mode", + options=[ + IO.DynamicCombo.Option(MODE_TEXT, []), + IO.DynamicCombo.Option( + MODE_AUDIO, + [ + IO.Audio.Input( + "reference_audio_1", + optional=True, + tooltip="Reference clip for voice cloning, tagged @Audio1 in the prompt. " + "Up to 30s.", + ), + IO.Audio.Input( + "reference_audio_2", + optional=True, + tooltip="Reference clip tagged @Audio2 in the prompt. Up to 30s.", + ), + IO.Audio.Input( + "reference_audio_3", + optional=True, + tooltip="Reference clip tagged @Audio3 in the prompt. Up to 30s.", + ), + ], + ), + IO.DynamicCombo.Option( + MODE_IMAGE, + [ + IO.Image.Input( + "reference_image", + optional=True, + tooltip="A single character image; the model derives a voice from it. " + "Cannot be combined with reference audio.", + ), + ], + ), + IO.DynamicCombo.Option( + MODE_SPEAKER, + [ + IO.Combo.Input( + "preset_voice", + options=SEED_AUDIO_VOICE_OPTIONS, + default=SEED_AUDIO_VOICE_OPTIONS[0], + tooltip="A built-in TTS 2.0 voice that reads the prompt. No reference " + "clip needed, and @AudioN tags are not used in this mode.", + ), + ], + ), + ], + tooltip=( + "How to condition the voice: 'text only' (describe everything in the prompt), " + "'audio reference' (clone up to 3 voices, tagged @Audio1-3), 'image reference' " + "(derive a voice from one character image), or 'preset voice' (pick a built-in " + "named voice that reads the prompt)." + ), + ), + IO.Combo.Input( + "sample_rate", + options=["8000", "16000", "24000", "32000", "44100", "48000"], + default="24000", + tooltip="Output sample rate in Hz.", + ), + IO.Int.Input( + "speech_rate", + default=0, + min=-50, + max=100, + tooltip="Speaking speed. 0 = normal, 100 = 2.0x, -50 = 0.5x.", + ), + IO.Int.Input( + "loudness_rate", + default=0, + min=-50, + max=100, + tooltip="Loudness. 0 = normal, 100 = 2.0x, -50 = 0.5x.", + ), + IO.Int.Input( + "pitch_rate", + default=0, + min=-12, + max=12, + tooltip="Pitch shift in semitones (-12 to 12).", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[IO.Audio.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd": 0.2145, "format":{"suffix":"/minute","approximate":true}}""", + ), + ) + + @classmethod + async def execute( + cls, + text_prompt: str, + reference_mode: dict, + sample_rate: str, + speech_rate: int, + loudness_rate: int, + pitch_rate: int, + seed: int, + ) -> IO.NodeOutput: + mode = reference_mode["reference_mode"] + audio_indices = connected_audio_indices(reference_mode) + image = reference_mode.get("reference_image") + preset_voice = reference_mode.get("preset_voice") + validate_seed_audio_inputs(text_prompt, mode, audio_indices, image is not None, preset_voice) + + references: list[SeedAudioReference] | None = None + if mode == MODE_AUDIO: + references = [] + for i in audio_indices: + clip = reference_mode[f"reference_audio_{i}"] + validate_audio_duration(clip, max_duration=30.0) + mp3_bytes = audio_input_to_mp3(clip).getvalue() + references.append(SeedAudioReference(audio_data=base64.b64encode(mp3_bytes).decode("utf-8"))) + elif mode == MODE_IMAGE: + image = upscale_image_tensor_to_min_pixels(image, 160_000) + references = [SeedAudioReference(image_data=tensor_to_base64_string(image, mime_type="image/png"))] + elif mode == MODE_SPEAKER: + references = [SeedAudioReference(speaker=SEED_AUDIO_VOICE_MAP[preset_voice])] + + response = await sync_op( + cls, + ApiEndpoint(path="/proxy/byteplus/api/v3/tts/create", method="POST"), + response_model=SeedAudioResponse, + data=SeedAudioRequest( + text_prompt=text_prompt, + references=references, + audio_config=SeedAudioConfig( + sample_rate=int(sample_rate), + speech_rate=speech_rate, + loudness_rate=loudness_rate, + pitch_rate=pitch_rate, + ), + ), + ) + if not response.audio: + raise Exception( + f"Seed Audio returned no audio (code={response.code}): {response.message}" + ) + return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response.audio))) + + class ByteDanceExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -2400,6 +2878,7 @@ class ByteDanceExtension(ComfyExtension): ByteDance2ReferenceNode, ByteDanceCreateImageAsset, ByteDanceCreateVideoAsset, + ByteDanceSeedAudioNode, ] diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.py index e75ef3835..a8eb0a797 100644 --- a/comfy_api_nodes/nodes_gemini.py +++ b/comfy_api_nodes/nodes_gemini.py @@ -5,20 +5,20 @@ See: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/infer import base64 import os -from enum import Enum from fnmatch import fnmatch from io import BytesIO -from typing import Literal +from typing import Any, Literal import torch from typing_extensions import override import folder_paths -from comfy_api.latest import IO, ComfyExtension, Input, Types +from comfy_api.latest import IO, ComfyExtension, Input, InputImpl, Types from comfy_api_nodes.apis.gemini import ( GeminiContent, GeminiFileData, GeminiGenerateContentRequest, + GeminiGenerationConfig, GeminiGenerateContentResponse, GeminiImageConfig, GeminiImageGenerateContentRequest, @@ -37,16 +37,23 @@ from comfy_api_nodes.util import ( audio_to_base64_string, bytesio_to_image_tensor, download_url_to_image_tensor, + download_url_to_video_output, get_number_of_images, sync_op, tensor_to_base64_string, + upload_audio_to_comfyapi, + upload_image_to_comfyapi, upload_images_to_comfyapi, + upload_video_to_comfyapi, validate_string, + validate_video_duration, video_to_base64_string, ) GEMINI_BASE_ENDPOINT = "/proxy/vertexai/gemini" GEMINI_MAX_INPUT_FILE_SIZE = 20 * 1024 * 1024 # 20 MB +GEMINI_URL_INPUT_BUDGET = 10 +GEMINI_MAX_INLINE_BYTES = 18 * 1024 * 1024 GEMINI_IMAGE_SYS_PROMPT = ( "You are an expert image-generation engine. You must ALWAYS produce an image.\n" "Interpret all user input—regardless of " @@ -72,15 +79,6 @@ GEMINI_IMAGE_2_PRICE_BADGE = IO.PriceBadge( ) -class GeminiImageModel(str, Enum): - """ - Gemini Image Model Names allowed by comfy-api - """ - - gemini_2_5_flash_image_preview = "gemini-2.5-flash-image-preview" - gemini_2_5_flash_image = "gemini-2.5-flash-image" - - async def create_image_parts( cls: type[IO.ComfyNode], images: Input.Image | list[Input.Image], @@ -233,25 +231,38 @@ async def get_image_from_response(response: GeminiGenerateContentResponse, thoug return torch.cat(image_tensors, dim=0) +async def get_video_from_response( + response: GeminiGenerateContentResponse, cls: type[IO.ComfyNode] | None = None +) -> InputImpl.VideoFromFile: + parts = get_parts_by_type(response, "video/*") + for part in parts: + if part.inlineData and part.inlineData.data: + return InputImpl.VideoFromFile(BytesIO(base64.b64decode(part.inlineData.data))) + if part.fileData and part.fileData.fileUri: + return await download_url_to_video_output(part.fileData.fileUri, cls=cls) + model_message = get_text_from_response(response).strip() + if model_message: + raise ValueError(f"Gemini did not generate a video. Model response: {model_message}") + raise ValueError( + "Gemini did not generate a video. Try rephrasing your prompt, " + "shortening the requested duration, or reducing the number of input images/videos." + ) + + def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | None: if not response.modelVersion: return None # Define prices (Cost per 1,000,000 tokens), see https://cloud.google.com/vertex-ai/generative-ai/pricing - if response.modelVersion in ("gemini-2.5-pro-preview-05-06", "gemini-2.5-pro"): + output_video_tokens_price = 0.0 + if response.modelVersion == "gemini-2.5-pro": input_tokens_price = 1.25 output_text_tokens_price = 10.0 output_image_tokens_price = 0.0 - elif response.modelVersion in ( - "gemini-2.5-flash-preview-04-17", - "gemini-2.5-flash", - ): + elif response.modelVersion == "gemini-2.5-flash": input_tokens_price = 0.30 output_text_tokens_price = 2.50 output_image_tokens_price = 0.0 - elif response.modelVersion in ( - "gemini-2.5-flash-image-preview", - "gemini-2.5-flash-image", - ): + elif response.modelVersion == "gemini-2.5-flash-image": input_tokens_price = 0.30 output_text_tokens_price = 2.50 output_image_tokens_price = 30.0 @@ -259,18 +270,27 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N input_tokens_price = 2 output_text_tokens_price = 12.0 output_image_tokens_price = 0.0 - elif response.modelVersion == "gemini-3.1-flash-lite-preview": + elif response.modelVersion in ("gemini-3.1-flash-lite-preview", "gemini-3.1-flash-lite"): input_tokens_price = 0.25 output_text_tokens_price = 1.50 output_image_tokens_price = 0.0 - elif response.modelVersion == "gemini-3-pro-image-preview": + elif response.modelVersion in ("gemini-3-pro-image-preview", "gemini-3-pro-image"): input_tokens_price = 2 output_text_tokens_price = 12.0 output_image_tokens_price = 120.0 - elif response.modelVersion == "gemini-3.1-flash-image-preview": + elif response.modelVersion in ("gemini-3.1-flash-image-preview", "gemini-3.1-flash-image"): input_tokens_price = 0.5 output_text_tokens_price = 3.0 output_image_tokens_price = 60.0 + elif response.modelVersion == "gemini-3.1-flash-lite-image": + input_tokens_price = 0.25 + output_text_tokens_price = 1.50 + output_image_tokens_price = 30.0 + elif response.modelVersion == "gemini-omni-flash-preview": + input_tokens_price = 2.145 + output_text_tokens_price = 12.87 + output_image_tokens_price = 0.0 + output_video_tokens_price = 25.025 else: return None final_price = response.usageMetadata.promptTokenCount * input_tokens_price @@ -278,6 +298,8 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N for i in response.usageMetadata.candidatesTokensDetails: if i.modality == Modality.IMAGE: final_price += output_image_tokens_price * i.tokenCount # for Nano Banana models + elif i.modality == Modality.VIDEO: + final_price += output_video_tokens_price * i.tokenCount # for Omni Flash else: final_price += output_text_tokens_price * i.tokenCount if response.usageMetadata.thoughtsTokenCount: @@ -285,6 +307,140 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N return final_price / 1_000_000.0 +def create_video_parts(video_input: Input.Video) -> list[GeminiPart]: + """Convert a single video input to Gemini API compatible parts (inline MP4/H.264).""" + base_64_string = video_to_base64_string( + video_input, container_format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264 + ) + return [ + GeminiPart( + inlineData=GeminiInlineData( + mimeType=GeminiMimeType.video_mp4, + data=base_64_string, + ) + ) + ] + + +def create_audio_parts(audio_input: Input.Audio) -> list[GeminiPart]: + """Convert an audio input to Gemini API compatible parts (one inline MP3 part per batch item).""" + audio_parts: list[GeminiPart] = [] + for batch_index in range(audio_input["waveform"].shape[0]): + # Recreate an IO.AUDIO object for the given batch dimension index + audio_at_index = Input.Audio( + waveform=audio_input["waveform"][batch_index].unsqueeze(0), + sample_rate=audio_input["sample_rate"], + ) + # Convert to MP3 format for compatibility with Gemini API + audio_bytes = audio_to_base64_string( + audio_at_index, + container_format="mp3", + codec_name="libmp3lame", + ) + audio_parts.append( + GeminiPart( + inlineData=GeminiInlineData( + mimeType=GeminiMimeType.audio_mp3, + data=audio_bytes, + ) + ) + ) + return audio_parts + + +def _flatten_images(images: list[Input.Image]) -> list[torch.Tensor]: + """Expand any batched image tensors into individual (H, W, C) frames, preserving order.""" + frames: list[torch.Tensor] = [] + for img in images: + if len(img.shape) == 4: + frames.extend(img[i] for i in range(img.shape[0])) + else: + frames.append(img) + return frames + + +def _flatten_audio(audios: list[Input.Audio]) -> list[Input.Audio]: + """Expand any batched audio inputs into individual single-clip audio inputs, preserving order.""" + clips: list[Input.Audio] = [] + for audio in audios: + waveform = audio["waveform"] + for i in range(waveform.shape[0]): + clips.append(Input.Audio(waveform=waveform[i].unsqueeze(0), sample_rate=audio["sample_rate"])) + return clips + + +async def _media_url_part(cls: type[IO.ComfyNode], kind: str, payload: Any) -> GeminiPart: + """Upload a single media unit to ComfyAPI storage and return a fileData (URL) part.""" + if kind == "image": + url = await upload_image_to_comfyapi(cls, payload, mime_type="image/png", wait_label="Uploading image") + return GeminiPart(fileData=GeminiFileData(mimeType=GeminiMimeType.image_png, fileUri=url)) + if kind == "audio": + url = await upload_audio_to_comfyapi( + cls, payload, container_format="mp3", codec_name="libmp3lame", mime_type="audio/mp3" + ) + return GeminiPart(fileData=GeminiFileData(mimeType=GeminiMimeType.audio_mp3, fileUri=url)) + url = await upload_video_to_comfyapi(cls, payload, wait_label="Uploading video") + return GeminiPart(fileData=GeminiFileData(mimeType=GeminiMimeType.video_mp4, fileUri=url)) + + +def _media_inline_part(kind: str, payload: Any) -> tuple[GeminiPart, int]: + """Encode a single media unit as an inline base64 part; returns (part, base64_length).""" + if kind == "image": + data = tensor_to_base64_string(payload, mime_type="image/webp") + mime = GeminiMimeType.image_webp + elif kind == "audio": + data = audio_to_base64_string(payload, container_format="mp3", codec_name="libmp3lame") + mime = GeminiMimeType.audio_mp3 + else: + data = video_to_base64_string( + payload, container_format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264 + ) + mime = GeminiMimeType.video_mp4 + return GeminiPart(inlineData=GeminiInlineData(mimeType=mime, data=data)), len(data) + + +async def build_gemini_media_parts( + cls: type[IO.ComfyNode], + images: list[Input.Image], + audios: list[Input.Audio], + videos: list[Input.Video], + *, + url_budget: int = GEMINI_URL_INPUT_BUDGET, + max_inline_bytes: int = GEMINI_MAX_INLINE_BYTES, +) -> list[GeminiPart]: + """Build Gemini parts for multimodal inputs (images, audio, video). + + fileData URLs are preferred for every media type: the upload is fetched directly by the + model, keeping the request body tiny regardless of media size. The URL budget is shared + across all media and assigned largest-first (video, then audio, then images), so that if it + is ever exhausted the inline-base64 overflow is limited to the smallest items. Total inline + payload is capped by `max_inline_bytes`. + """ + units: list[tuple[str, Any]] = ( + [("video", v) for v in videos] + + [("audio", a) for a in _flatten_audio(audios)] + + [("image", f) for f in _flatten_images(images)] + ) + + parts: list[GeminiPart] = [] + url_used = 0 + inline_bytes = 0 + for kind, payload in units: + if url_used < url_budget: + parts.append(await _media_url_part(cls, kind, payload)) + url_used += 1 + continue + part, nbytes = _media_inline_part(kind, payload) + inline_bytes += nbytes + if inline_bytes > max_inline_bytes: + raise ValueError( + f"Too much media to send inline (over {max_inline_bytes // (1024 * 1024)}MB after the first " + f"{url_budget} inputs are uploaded as URLs). Reduce the number or size of attached media." + ) + parts.append(part) + return parts + + class GeminiNode(IO.ComfyNode): """ Node to generate text responses from a Gemini model. @@ -315,8 +471,6 @@ class GeminiNode(IO.ComfyNode): IO.Combo.Input( "model", options=[ - "gemini-2.5-pro-preview-05-06", - "gemini-2.5-flash-preview-04-17", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro-preview", @@ -407,58 +561,9 @@ class GeminiNode(IO.ComfyNode): ) """, ), + is_deprecated=True, ) - @classmethod - def create_video_parts(cls, video_input: Input.Video) -> list[GeminiPart]: - """Convert video input to Gemini API compatible parts.""" - - base_64_string = video_to_base64_string( - video_input, container_format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264 - ) - return [ - GeminiPart( - inlineData=GeminiInlineData( - mimeType=GeminiMimeType.video_mp4, - data=base_64_string, - ) - ) - ] - - @classmethod - def create_audio_parts(cls, audio_input: Input.Audio) -> list[GeminiPart]: - """ - Convert audio input to Gemini API compatible parts. - - Args: - audio_input: Audio input from ComfyUI, containing waveform tensor and sample rate. - - Returns: - List of GeminiPart objects containing the encoded audio. - """ - audio_parts: list[GeminiPart] = [] - for batch_index in range(audio_input["waveform"].shape[0]): - # Recreate an IO.AUDIO object for the given batch dimension index - audio_at_index = Input.Audio( - waveform=audio_input["waveform"][batch_index].unsqueeze(0), - sample_rate=audio_input["sample_rate"], - ) - # Convert to MP3 format for compatibility with Gemini API - audio_bytes = audio_to_base64_string( - audio_at_index, - container_format="mp3", - codec_name="libmp3lame", - ) - audio_parts.append( - GeminiPart( - inlineData=GeminiInlineData( - mimeType=GeminiMimeType.audio_mp3, - data=audio_bytes, - ) - ) - ) - return audio_parts - @classmethod async def execute( cls, @@ -482,9 +587,9 @@ class GeminiNode(IO.ComfyNode): if images is not None: parts.extend(await create_image_parts(cls, images)) if audio is not None: - parts.extend(cls.create_audio_parts(audio)) + parts.extend(create_audio_parts(audio)) if video is not None: - parts.extend(cls.create_video_parts(video)) + parts.extend(create_video_parts(video)) if files is not None: parts.extend(files) @@ -512,6 +617,210 @@ class GeminiNode(IO.ComfyNode): return IO.NodeOutput(output_text or "Empty response from Gemini model...") +GEMINI_V2_MODELS: dict[str, str] = { + "Gemini 3.1 Pro": "gemini-3.1-pro-preview", + "Gemini 3.1 Flash-Lite": "gemini-3.1-flash-lite-preview", +} + + +def _gemini_text_model_inputs(thinking_default: str) -> list[Input]: + """Per-model inputs revealed by the model DynamicCombo (shared media + sampling controls).""" + return [ + IO.Autogrow.Input( + "images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"image_{i}" for i in range(1, 17)], + min=0, + ), + tooltip="Optional image(s) to use as context for the model. Up to 16 images.", + ), + IO.Autogrow.Input( + "audio", + template=IO.Autogrow.TemplateNames( + IO.Audio.Input("audio"), + names=["audio_1"], + min=0, + ), + tooltip="Optional audio clip to use as context for the model.", + ), + IO.Autogrow.Input( + "video", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("video"), + names=["video_1"], + min=0, + ), + tooltip="Optional video clip to use as context for the model.", + ), + IO.Custom("GEMINI_INPUT_FILES").Input( + "files", + optional=True, + tooltip="Optional file(s) to use as context for the model. " + "Accepts inputs from the Gemini Input Files node.", + ), + IO.Combo.Input( + "thinking_level", + options=["LOW", "HIGH"], + default=thinking_default, + tooltip="How hard the model reasons internally before answering. " + "HIGH improves quality on difficult tasks but costs more (thinking) tokens and is slower.", + ), + IO.Float.Input( + "temperature", + default=1.0, + min=0.0, + max=2.0, + step=0.01, + tooltip="Controls randomness. Lower is more focused/deterministic, higher is more creative.", + advanced=True, + ), + IO.Float.Input( + "top_p", + default=0.95, + min=0.0, + max=1.0, + step=0.01, + tooltip="Nucleus sampling: sample from the smallest token set whose cumulative probability reaches top_p.", + advanced=True, + ), + IO.Int.Input( + "max_output_tokens", + default=32768, + min=16, + max=65536, + tooltip="Maximum tokens to generate, including the model's internal thinking. " + "With thinking_level HIGH, a low value can leave no room for the answer; raise this if " + "responses come back empty or truncated. The model stops early when finished, so a higher " + "cap costs nothing extra for short replies.", + advanced=True, + ), + ] + + +class GeminiNodeV2(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="GeminiNodeV2", + display_name="Google Gemini", + category="partner/text/Gemini", + essentials_category="Text Generation", + description="Generate text responses with Google's Gemini models. Provide a text prompt and, " + "optionally, one or more images, audio clips, videos, or files as multimodal context.", + inputs=[ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Text input to the model. Include detailed instructions, questions, or context.", + ), + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option("Gemini 3.1 Pro", _gemini_text_model_inputs("HIGH")), + IO.DynamicCombo.Option("Gemini 3.1 Flash-Lite", _gemini_text_model_inputs("LOW")), + ], + tooltip="The Gemini model used to generate the response.", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed for sampling. Set to 0 for a random seed. Deterministic output isn't guaranteed.", + ), + IO.String.Input( + "system_prompt", + multiline=True, + default="", + optional=True, + advanced=True, + tooltip="Foundational instructions that dictate the model's behavior.", + ), + ], + outputs=[ + IO.String.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["model"]), + expr=""" + ( + $m := widgets.model; + $contains($m, "lite") ? { + "type": "list_usd", + "usd": [0.00025, 0.0015], + "format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" } + } : { + "type": "list_usd", + "usd": [0.002, 0.012], + "format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" } + } + ) + """, + ), + ) + + @classmethod + async def execute( + cls, + prompt: str, + model: dict, + seed: int, + system_prompt: str = "", + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1) + model_id = GEMINI_V2_MODELS[model["model"]] + + parts: list[GeminiPart] = [GeminiPart(text=prompt)] + images = [t for t in (model.get("images") or {}).values() if t is not None] + audios = [a for a in (model.get("audio") or {}).values() if a is not None] + videos = [v for v in (model.get("video") or {}).values() if v is not None] + if images or audios or videos: + parts.extend(await build_gemini_media_parts(cls, images, audios, videos)) + files = model.get("files") + if files is not None: + parts.extend(files) + + gemini_system_prompt = None + if system_prompt: + gemini_system_prompt = GeminiSystemInstructionContent(parts=[GeminiTextPart(text=system_prompt)], role=None) + + response = await sync_op( + cls, + endpoint=ApiEndpoint(path=f"{GEMINI_BASE_ENDPOINT}/{model_id}", method="POST"), + data=GeminiGenerateContentRequest( + contents=[ + GeminiContent( + role=GeminiRole.user, + parts=parts, + ) + ], + generationConfig=GeminiGenerationConfig( + temperature=model["temperature"], + topP=model["top_p"], + maxOutputTokens=model["max_output_tokens"], + seed=seed if seed > 0 else None, + thinkingConfig=GeminiThinkingConfig(thinkingLevel=model["thinking_level"]), + ), + systemInstruction=gemini_system_prompt, + ), + response_model=GeminiGenerateContentResponse, + price_extractor=calculate_tokens_price, + ) + + output_text = get_text_from_response(response) + return IO.NodeOutput(output_text or "Empty response from Gemini model...") + + class GeminiInputFiles(IO.ComfyNode): """ Loads and formats input files for use with the Gemini API. @@ -609,8 +918,7 @@ class GeminiImage(IO.ComfyNode): ), IO.Combo.Input( "model", - options=GeminiImageModel, - default=GeminiImageModel.gemini_2_5_flash_image, + options=["gemini-2.5-flash-image"], tooltip="The Gemini model to use for generating responses.", ), IO.Int.Input( @@ -825,7 +1133,9 @@ class GeminiImage2(IO.ComfyNode): ) -> IO.NodeOutput: validate_string(prompt, strip_whitespace=True, min_length=1) if model == "Nano Banana 2 (Gemini 3.1 Flash Image)": - model = "gemini-3.1-flash-image-preview" + model = "gemini-3.1-flash-image" + elif model == "gemini-3-pro-image-preview": + model = "gemini-3-pro-image" parts: list[GeminiPart] = [GeminiPart(text=prompt)] if images is not None: @@ -1026,7 +1336,7 @@ class GeminiNanoBanana2(IO.ComfyNode): ) -def _nano_banana_2_v2_model_inputs(): +def _nano_banana_2_v2_model_inputs(resolutions: list[str]): return [ IO.Combo.Input( "aspect_ratio", @@ -1053,8 +1363,8 @@ def _nano_banana_2_v2_model_inputs(): ), IO.Combo.Input( "resolution", - options=["1K", "2K", "4K"], - tooltip="Target output resolution. For 2K/4K the native Gemini upscaler is used.", + options=resolutions, + tooltip="Target output resolution.", ), IO.Combo.Input( "thinking_level", @@ -1100,7 +1410,11 @@ class GeminiNanoBanana2V2(IO.ComfyNode): options=[ IO.DynamicCombo.Option( "Nano Banana 2 (Gemini 3.1 Flash Image)", - _nano_banana_2_v2_model_inputs(), + _nano_banana_2_v2_model_inputs(resolutions=["1K", "2K", "4K"]), + ), + IO.DynamicCombo.Option( + "Nano Banana 2 Lite", + _nano_banana_2_v2_model_inputs(resolutions=["1K"]), ), ], ), @@ -1129,6 +1443,26 @@ class GeminiNanoBanana2V2(IO.ComfyNode): tooltip="Foundational instructions that dictate an AI's behavior.", advanced=True, ), + IO.Float.Input( + "temperature", + default=1.0, + min=0.0, + max=2.0, + step=0.01, + optional=True, + tooltip="Controls randomness in generation. Lower is more focused/deterministic.", + advanced=True, + ), + IO.Float.Input( + "top_p", + default=0.95, + min=0.0, + max=1.0, + step=0.01, + optional=True, + tooltip="Nucleus sampling threshold. Lower is more focused, higher more diverse.", + advanced=True, + ), ], outputs=[ IO.Image.Output(), @@ -1149,9 +1483,13 @@ class GeminiNanoBanana2V2(IO.ComfyNode): depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution"]), expr=""" ( - $r := $lookup(widgets, "model.resolution"); - $prices := {"1k": 0.0696, "2k": 0.1014, "4k": 0.154}; - {"type":"usd","usd": $lookup($prices, $r), "format":{"suffix":"/Image","approximate":true}} + $contains(widgets.model, "lite") + ? {"type":"usd","usd": 0.034, "format":{"suffix":"/Image","approximate":true}} + : ( + $r := $lookup(widgets, "model.resolution"); + $prices := {"1k": 0.0696, "2k": 0.1014, "4k": 0.154}; + {"type":"usd","usd": $lookup($prices, $r), "format":{"suffix":"/Image","approximate":true}} + ) ) """, ), @@ -1165,11 +1503,15 @@ class GeminiNanoBanana2V2(IO.ComfyNode): seed: int, response_modalities: str, system_prompt: str = "", + temperature: float = 1.0, + top_p: float = 0.95, ) -> IO.NodeOutput: validate_string(prompt, strip_whitespace=True, min_length=1) model_choice = model["model"] if model_choice == "Nano Banana 2 (Gemini 3.1 Flash Image)": - model_id = "gemini-3.1-flash-image-preview" + model_id = "gemini-3.1-flash-image" + elif model_choice == "Nano Banana 2 Lite": + model_id = "gemini-3.1-flash-lite-image" else: model_id = model_choice @@ -1204,6 +1546,8 @@ class GeminiNanoBanana2V2(IO.ComfyNode): responseModalities=(["IMAGE"] if response_modalities == "IMAGE" else ["TEXT", "IMAGE"]), imageConfig=image_config, thinkingConfig=GeminiThinkingConfig(thinkingLevel=model["thinking_level"]), + temperature=temperature, + topP=top_p, ), systemInstruction=gemini_system_prompt, ), @@ -1217,15 +1561,160 @@ class GeminiNanoBanana2V2(IO.ComfyNode): ) +OMNI_MAX_IMAGES = 14 +OMNI_MAX_VIDEOS = 3 + +OMNI_MODELS: dict[str, str] = { + "Omni Flash": "gemini-omni-flash-preview", +} + + +def _omni_flash_inputs() -> list[Input]: + """Per-model inputs for the Omni video DynamicCombo (prompt + reference media + sampling).""" + return [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describe the video to generate. Specify the length and aspect ratio directly in the " + 'prompt, e.g. "a 6-second clip in 16:9". Length may be 3-10 seconds; the aspect ratio must be ' + "16:9 (landscape) or 9:16 (portrait). The output is 720p, 24 FPS, with audio.", + ), + IO.Autogrow.Input( + "images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)], + min=0, + ), + tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images.", + ), + IO.Autogrow.Input( + "videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("video"), + names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)], + min=0, + ), + tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, " + f"each up to 10 seconds long.", + ), + IO.Float.Input( + "temperature", + default=1.0, + min=0.0, + max=2.0, + step=0.01, + tooltip="Controls randomness. Lower is more focused/deterministic, higher is more varied.", + advanced=True, + ), + IO.Float.Input( + "top_p", + default=0.95, + min=0.0, + max=1.0, + step=0.01, + tooltip="Nucleus sampling: sample from the smallest token set whose cumulative probability reaches top_p.", + advanced=True, + ), + ] + + +class GeminiVideoOmni(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="GeminiVideoOmni", + display_name="Google Gemini Omni (Video)", + category="partner/video/Gemini", + essentials_category="Video Generation", + description="Generate a video with audio from a text prompt using Google's Gemini Omni Flash model. " + "Optionally provide reference images and/or videos to guide or edit the result. Describe the desired " + "length (3-10s) and aspect ratio (16:9 or 9:16) directly in the prompt.", + inputs=[ + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option("Omni Flash", _omni_flash_inputs()), + ], + tooltip="The Gemini video model used to generate the video.", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr='{"type":"usd","usd":0.146,"format":{"suffix":"/second","approximate":true}}' + ), + ) + + @classmethod + async def execute(cls, model: dict, seed: int) -> IO.NodeOutput: + prompt = model.get("prompt") or "" + validate_string(prompt, strip_whitespace=True, min_length=1) + model_id = OMNI_MODELS[model["model"]] + + images = [t for t in (model.get("images") or {}).values() if t is not None] + videos = [v for v in (model.get("videos") or {}).values() if v is not None] + if sum(get_number_of_images(t) for t in images) > OMNI_MAX_IMAGES: + raise ValueError(f"The current maximum number of supported images is {OMNI_MAX_IMAGES}.") + if len(videos) > OMNI_MAX_VIDEOS: + raise ValueError(f"The current maximum number of supported videos is {OMNI_MAX_VIDEOS}.") + for video in videos: + validate_video_duration(video, max_duration=10) + + parts: list[GeminiPart] = [] + if images or videos: + parts.extend(await build_gemini_media_parts(cls, images, [], videos)) + parts.append(GeminiPart(text=prompt)) + response = await sync_op( + cls, + ApiEndpoint(path=f"{GEMINI_BASE_ENDPOINT}/{model_id}", method="POST"), + data=GeminiGenerateContentRequest( + contents=[GeminiContent(role=GeminiRole.user, parts=parts)], + generationConfig=GeminiGenerationConfig( + responseModalities=["TEXT", "VIDEO"], + temperature=model.get("temperature", 1.0), + topP=model.get("top_p", 0.95), + ), + ), + response_model=GeminiGenerateContentResponse, + price_extractor=calculate_tokens_price, + ) + return IO.NodeOutput( + await get_video_from_response(response, cls=cls), + get_text_from_response(response), + ) + + class GeminiExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: return [ GeminiNode, + GeminiNodeV2, GeminiImage, GeminiImage2, GeminiNanoBanana2, GeminiNanoBanana2V2, + GeminiVideoOmni, GeminiInputFiles, ] diff --git a/comfy_api_nodes/nodes_grok.py b/comfy_api_nodes/nodes_grok.py index 2ae529813..dc484536e 100644 --- a/comfy_api_nodes/nodes_grok.py +++ b/comfy_api_nodes/nodes_grok.py @@ -30,7 +30,7 @@ from comfy_api_nodes.util import ( _GROK_VIDEO_MODEL_API_IDS = { - "grok-imagine-video-1.5": "grok-imagine-video-1.5-preview", + "grok-imagine-video-1.5": "grok-imagine-video-1.5", } @@ -521,8 +521,8 @@ class GrokVideoNode(IO.ComfyNode): ), IO.Combo.Input( "resolution", - options=["480p", "720p"], - tooltip="The resolution of the output video.", + options=["480p", "720p", "1080p"], + tooltip="The resolution of the output video. 1080p is only available for grok-imagine-video-1.5.", ), IO.Combo.Input( "aspect_ratio", @@ -570,11 +570,12 @@ class GrokVideoNode(IO.ComfyNode): ( $is15 := $contains(widgets.model, "1.5"); $rate := $is15 - ? (widgets.resolution = "720p" ? 0.2002 : 0.1144) + ? (widgets.resolution = "1080p" ? 0.25 : (widgets.resolution = "720p" ? 0.14 : 0.08)) : (widgets.resolution = "720p" ? 0.07 : 0.05); - $imgCost := $is15 ? 0.0143 : 0.002; + $imgCost := $is15 ? 0.01 : 0.002; $base := $rate * widgets.duration; - {"type":"usd","usd": inputs.image.connected ? $base + $imgCost : $base} + $total := inputs.image.connected ? $base + $imgCost : $base; + {"type":"usd","usd": $is15 ? $total * 1.43 : $total} ) """, ), @@ -593,6 +594,8 @@ class GrokVideoNode(IO.ComfyNode): ) -> IO.NodeOutput: if image is None and model == "grok-imagine-video-1.5": raise ValueError(f"The '{model}' model requires an input image; connect one to the 'image' input.") + if resolution == "1080p" and model != "grok-imagine-video-1.5": + raise ValueError(f"1080p resolution is only available for grok-imagine-video-1.5, not '{model}'.") image_url = None if image is not None: if get_number_of_images(image) != 1: diff --git a/comfy_api_nodes/nodes_hunyuan3d.py b/comfy_api_nodes/nodes_hunyuan3d.py index fcd27b7fb..a9942476c 100644 --- a/comfy_api_nodes/nodes_hunyuan3d.py +++ b/comfy_api_nodes/nodes_hunyuan3d.py @@ -642,6 +642,7 @@ class Tencent3DPartNode(IO.ComfyNode): response_model=To3DProTaskCreateResponse, data=To3DPartTaskRequest( File=TaskFile3DInput(Type=file_format.upper(), Url=model_url), + EnableStagedGeneration=True, ), is_rate_limited=_is_tencent_rate_limited, ) diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 8018c3902..cc0467987 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -5,11 +5,10 @@ from PIL import Image import numpy as np import torch from comfy_api_nodes.apis.ideogram import ( - IdeogramGenerateRequest, IdeogramGenerateResponse, - ImageRequest, IdeogramV3Request, IdeogramV3EditRequest, + IdeogramV4Request, ) from comfy_api_nodes.util import ( ApiEndpoint, @@ -17,103 +16,9 @@ from comfy_api_nodes.util import ( download_url_as_bytesio, resize_mask_to_image, sync_op, + validate_string, ) -V1_V1_RES_MAP = { - "Auto":"AUTO", - "512 x 1536":"RESOLUTION_512_1536", - "576 x 1408":"RESOLUTION_576_1408", - "576 x 1472":"RESOLUTION_576_1472", - "576 x 1536":"RESOLUTION_576_1536", - "640 x 1024":"RESOLUTION_640_1024", - "640 x 1344":"RESOLUTION_640_1344", - "640 x 1408":"RESOLUTION_640_1408", - "640 x 1472":"RESOLUTION_640_1472", - "640 x 1536":"RESOLUTION_640_1536", - "704 x 1152":"RESOLUTION_704_1152", - "704 x 1216":"RESOLUTION_704_1216", - "704 x 1280":"RESOLUTION_704_1280", - "704 x 1344":"RESOLUTION_704_1344", - "704 x 1408":"RESOLUTION_704_1408", - "704 x 1472":"RESOLUTION_704_1472", - "720 x 1280":"RESOLUTION_720_1280", - "736 x 1312":"RESOLUTION_736_1312", - "768 x 1024":"RESOLUTION_768_1024", - "768 x 1088":"RESOLUTION_768_1088", - "768 x 1152":"RESOLUTION_768_1152", - "768 x 1216":"RESOLUTION_768_1216", - "768 x 1232":"RESOLUTION_768_1232", - "768 x 1280":"RESOLUTION_768_1280", - "768 x 1344":"RESOLUTION_768_1344", - "832 x 960":"RESOLUTION_832_960", - "832 x 1024":"RESOLUTION_832_1024", - "832 x 1088":"RESOLUTION_832_1088", - "832 x 1152":"RESOLUTION_832_1152", - "832 x 1216":"RESOLUTION_832_1216", - "832 x 1248":"RESOLUTION_832_1248", - "864 x 1152":"RESOLUTION_864_1152", - "896 x 960":"RESOLUTION_896_960", - "896 x 1024":"RESOLUTION_896_1024", - "896 x 1088":"RESOLUTION_896_1088", - "896 x 1120":"RESOLUTION_896_1120", - "896 x 1152":"RESOLUTION_896_1152", - "960 x 832":"RESOLUTION_960_832", - "960 x 896":"RESOLUTION_960_896", - "960 x 1024":"RESOLUTION_960_1024", - "960 x 1088":"RESOLUTION_960_1088", - "1024 x 640":"RESOLUTION_1024_640", - "1024 x 768":"RESOLUTION_1024_768", - "1024 x 832":"RESOLUTION_1024_832", - "1024 x 896":"RESOLUTION_1024_896", - "1024 x 960":"RESOLUTION_1024_960", - "1024 x 1024":"RESOLUTION_1024_1024", - "1088 x 768":"RESOLUTION_1088_768", - "1088 x 832":"RESOLUTION_1088_832", - "1088 x 896":"RESOLUTION_1088_896", - "1088 x 960":"RESOLUTION_1088_960", - "1120 x 896":"RESOLUTION_1120_896", - "1152 x 704":"RESOLUTION_1152_704", - "1152 x 768":"RESOLUTION_1152_768", - "1152 x 832":"RESOLUTION_1152_832", - "1152 x 864":"RESOLUTION_1152_864", - "1152 x 896":"RESOLUTION_1152_896", - "1216 x 704":"RESOLUTION_1216_704", - "1216 x 768":"RESOLUTION_1216_768", - "1216 x 832":"RESOLUTION_1216_832", - "1232 x 768":"RESOLUTION_1232_768", - "1248 x 832":"RESOLUTION_1248_832", - "1280 x 704":"RESOLUTION_1280_704", - "1280 x 720":"RESOLUTION_1280_720", - "1280 x 768":"RESOLUTION_1280_768", - "1280 x 800":"RESOLUTION_1280_800", - "1312 x 736":"RESOLUTION_1312_736", - "1344 x 640":"RESOLUTION_1344_640", - "1344 x 704":"RESOLUTION_1344_704", - "1344 x 768":"RESOLUTION_1344_768", - "1408 x 576":"RESOLUTION_1408_576", - "1408 x 640":"RESOLUTION_1408_640", - "1408 x 704":"RESOLUTION_1408_704", - "1472 x 576":"RESOLUTION_1472_576", - "1472 x 640":"RESOLUTION_1472_640", - "1472 x 704":"RESOLUTION_1472_704", - "1536 x 512":"RESOLUTION_1536_512", - "1536 x 576":"RESOLUTION_1536_576", - "1536 x 640":"RESOLUTION_1536_640", -} - -V1_V2_RATIO_MAP = { - "1:1":"ASPECT_1_1", - "4:3":"ASPECT_4_3", - "3:4":"ASPECT_3_4", - "16:9":"ASPECT_16_9", - "9:16":"ASPECT_9_16", - "2:1":"ASPECT_2_1", - "1:2":"ASPECT_1_2", - "3:2":"ASPECT_3_2", - "2:3":"ASPECT_2_3", - "4:5":"ASPECT_4_5", - "5:4":"ASPECT_5_4", -} V3_RATIO_MAP = { "1:3":"1x3", @@ -227,298 +132,6 @@ async def download_and_process_images(image_urls): return stacked_tensors -class IdeogramV1(IO.ComfyNode): - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="IdeogramV1", - display_name="Ideogram V1", - category="partner/image/Ideogram", - description="Generates images using the Ideogram V1 model.", - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="Prompt for the image generation", - ), - IO.Boolean.Input( - "turbo", - default=False, - tooltip="Whether to use turbo mode (faster generation, potentially lower quality)", - ), - IO.Combo.Input( - "aspect_ratio", - options=list(V1_V2_RATIO_MAP.keys()), - default="1:1", - tooltip="The aspect ratio for image generation.", - optional=True, - ), - IO.Combo.Input( - "magic_prompt_option", - options=["AUTO", "ON", "OFF"], - default="AUTO", - tooltip="Determine if MagicPrompt should be used in generation", - optional=True, - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=2147483647, - step=1, - control_after_generate=True, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - IO.String.Input( - "negative_prompt", - multiline=True, - default="", - tooltip="Description of what to exclude from the image", - optional=True, - ), - IO.Int.Input( - "num_images", - default=1, - min=1, - max=8, - step=1, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["num_images", "turbo"]), - expr=""" - ( - $n := widgets.num_images; - $base := (widgets.turbo = true) ? 0.0286 : 0.0858; - {"type":"usd","usd": $round($base * $n, 2)} - ) - """, - ), - ) - - @classmethod - async def execute( - cls, - prompt, - turbo=False, - aspect_ratio="1:1", - magic_prompt_option="AUTO", - seed=0, - negative_prompt="", - num_images=1, - ): - # Determine the model based on turbo setting - aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None) - model = "V_1_TURBO" if turbo else "V_1" - - response = await sync_op( - cls, - ApiEndpoint(path="/proxy/ideogram/generate", method="POST"), - response_model=IdeogramGenerateResponse, - data=IdeogramGenerateRequest( - image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, - magic_prompt_option=(magic_prompt_option if magic_prompt_option != "AUTO" else None), - negative_prompt=negative_prompt if negative_prompt else None, - ) - ), - max_retries=1, - ) - - if not response.data or len(response.data) == 0: - raise Exception("No images were generated in the response") - - image_urls = [image_data.url for image_data in response.data if image_data.url] - if not image_urls: - raise Exception("No image URLs were generated in the response") - return IO.NodeOutput(await download_and_process_images(image_urls)) - - -class IdeogramV2(IO.ComfyNode): - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="IdeogramV2", - display_name="Ideogram V2", - category="partner/image/Ideogram", - description="Generates images using the Ideogram V2 model.", - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="Prompt for the image generation", - ), - IO.Boolean.Input( - "turbo", - default=False, - tooltip="Whether to use turbo mode (faster generation, potentially lower quality)", - ), - IO.Combo.Input( - "aspect_ratio", - options=list(V1_V2_RATIO_MAP.keys()), - default="1:1", - tooltip="The aspect ratio for image generation. Ignored if resolution is not set to AUTO.", - optional=True, - ), - IO.Combo.Input( - "resolution", - options=list(V1_V1_RES_MAP.keys()), - default="Auto", - tooltip="The resolution for image generation. " - "If not set to AUTO, this overrides the aspect_ratio setting.", - optional=True, - ), - IO.Combo.Input( - "magic_prompt_option", - options=["AUTO", "ON", "OFF"], - default="AUTO", - tooltip="Determine if MagicPrompt should be used in generation", - optional=True, - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=2147483647, - step=1, - control_after_generate=True, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - IO.Combo.Input( - "style_type", - options=["AUTO", "GENERAL", "REALISTIC", "DESIGN", "RENDER_3D", "ANIME"], - default="NONE", - tooltip="Style type for generation (V2 only)", - optional=True, - advanced=True, - ), - IO.String.Input( - "negative_prompt", - multiline=True, - default="", - tooltip="Description of what to exclude from the image", - optional=True, - ), - IO.Int.Input( - "num_images", - default=1, - min=1, - max=8, - step=1, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - #"color_palette": ( - # IO.STRING, - # { - # "multiline": False, - # "default": "", - # "tooltip": "Color palette preset name or hex colors with weights", - # }, - #), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["num_images", "turbo"]), - expr=""" - ( - $n := widgets.num_images; - $base := (widgets.turbo = true) ? 0.0715 : 0.1144; - {"type":"usd","usd": $round($base * $n, 2)} - ) - """, - ), - ) - - @classmethod - async def execute( - cls, - prompt, - turbo=False, - aspect_ratio="1:1", - resolution="Auto", - magic_prompt_option="AUTO", - seed=0, - style_type="NONE", - negative_prompt="", - num_images=1, - color_palette="", - ): - aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None) - resolution = V1_V1_RES_MAP.get(resolution, None) - # Determine the model based on turbo setting - model = "V_2_TURBO" if turbo else "V_2" - - # Handle resolution vs aspect_ratio logic - # If resolution is not AUTO, it overrides aspect_ratio - final_resolution = None - final_aspect_ratio = None - - if resolution != "AUTO": - final_resolution = resolution - else: - final_aspect_ratio = aspect_ratio if aspect_ratio != "ASPECT_1_1" else None - - response = await sync_op( - cls, - endpoint=ApiEndpoint(path="/proxy/ideogram/generate", method="POST"), - response_model=IdeogramGenerateResponse, - data=IdeogramGenerateRequest( - image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=final_aspect_ratio, - resolution=final_resolution, - magic_prompt_option=(magic_prompt_option if magic_prompt_option != "AUTO" else None), - style_type=style_type if style_type != "NONE" else None, - negative_prompt=negative_prompt if negative_prompt else None, - color_palette=color_palette if color_palette else None, - ) - ), - max_retries=1, - ) - if not response.data or len(response.data) == 0: - raise Exception("No images were generated in the response") - - image_urls = [image_data.url for image_data in response.data if image_data.url] - if not image_urls: - raise Exception("No image URLs were generated in the response") - return IO.NodeOutput(await download_and_process_images(image_urls)) - - class IdeogramV3(IO.ComfyNode): @classmethod @@ -798,13 +411,125 @@ class IdeogramV3(IO.ComfyNode): return IO.NodeOutput(await download_and_process_images(image_urls)) +class IdeogramV4(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="IdeogramV4", + display_name="Ideogram V4", + category="partner/image/Ideogram", + description="Generates images using the Ideogram 4.0 model from a text prompt.", + inputs=[ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Text prompt for the image generation.", + ), + IO.Combo.Input( + "resolution", + options=[ + "Auto", + "2048x2048 (1:1)", + "1440x2880 (1:2)", + "2880x1440 (2:1)", + "1664x2496 (2:3)", + "2496x1664 (3:2)", + "1792x2240 (4:5)", + "2240x1792 (5:4)", + "1440x2560 (9:16)", + "2560x1440 (16:9)", + "1600x2560 (5:8)", + "2560x1600 (8:5)", + "1728x2304 (3:4)", + "2304x1728 (4:3)", + "1296x3168 (9:22)", + "3168x1296 (22:9)", + "1152x2944 (9:23)", + "2944x1152 (23:9)", + "1248x3328 (3:8)", + "3328x1248 (8:3)", + "1280x3072 (5:12)", + "3072x1280 (12:5)", + ], + default="Auto", + ), + IO.Combo.Input( + "rendering_speed", + options=["DEFAULT", "TURBO", "QUALITY"], + default="DEFAULT", + tooltip="Controls the trade-off between generation speed and quality.", + ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=2147483647, + step=1, + control_after_generate=True, + display_mode=IO.NumberDisplay.number, + ), + ], + outputs=[ + IO.Image.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["rendering_speed"]), + expr=""" + ( + $speed := widgets.rendering_speed; + $price := + $contains($speed,"turbo") ? 0.0429 : + $contains($speed,"quality") ? 0.143 : + 0.0858; + {"type":"usd","usd": $price} + ) + """, + ), + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + rendering_speed: str, + seed: int, + ): + validate_string(prompt, strip_whitespace=True, min_length=1) + response = await sync_op( + cls, + ApiEndpoint(path="/proxy/ideogram/ideogram-v4/generate", method="POST"), + response_model=IdeogramGenerateResponse, + data=IdeogramV4Request( + text_prompt=prompt, + resolution=resolution.split(" ")[0] if resolution != "Auto" else None, + rendering_speed=rendering_speed, + ), + max_retries=1, + ) + + if not response.data or len(response.data) == 0: + raise Exception("No images were generated in the response") + image_urls = [image_data.url for image_data in response.data if image_data.url] + if not image_urls: + raise Exception("No image URLs were generated in the response") + return IO.NodeOutput(await download_and_process_images(image_urls)) + + class IdeogramExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: return [ - IdeogramV1, - IdeogramV2, IdeogramV3, + IdeogramV4, ] diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index d11e42540..b27de2549 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -60,6 +60,12 @@ from comfy_api_nodes.apis.kling import ( OmniProImageRequest, OmniProReferences2VideoRequest, OmniProText2VideoRequest, + Kling3TurboSettings, + Kling3TurboText2VideoRequest, + Kling3TurboContent, + Kling3TurboImage2VideoRequest, + Kling3TurboCreateResponse, + Kling3TurboQueryResponse, TaskStatusResponse, TextToVideoWithAudioRequest, ) @@ -436,7 +442,7 @@ async def execute_text2video( negative_prompt=negative_prompt if negative_prompt else None, duration=KlingVideoGenDuration(duration), mode=KlingVideoGenMode(model_mode), - model_name=KlingVideoGenModelName(model_name), + model_name=model_name, cfg_scale=cfg_scale, aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), camera_control=camera_control, @@ -2847,6 +2853,67 @@ class MotionControl(IO.ComfyNode): return IO.NodeOutput(await download_url_to_video_output(final_response.data.task_result.videos[0].url)) +def build_turbo_shot_prompt(multi_prompt: list[MultiPromptEntry]) -> str: + """Render storyboard entries into the Turbo multi-shot prompt 'shot n, m, words; ...'.""" + return "; ".join(f"shot {i}, {int(e.duration)}, {e.prompt}" for i, e in enumerate(multi_prompt, 1)) + ";" + + +def _turbo_video_url(response: Kling3TurboQueryResponse) -> str: + """Extract the result video URL from a /tasks response (data[].outputs[] where type == 'video').""" + task = response.data[0] if response.data else None + if task and task.outputs: + for output in task.outputs: + if output.type == "video" and output.url: + return output.url + raise RuntimeError(f"Kling 3.0 Turbo task finished without a video output: {response.model_dump()}") + + +async def execute_kling_turbo( + cls: type[IO.ComfyNode], + *, + prompt: str, + resolution: str, + aspect_ratio: str, + duration: int, + start_frame: torch.Tensor | None, +) -> IO.NodeOutput: + """Create + poll a Kling 3.0 Turbo task. Image-to-video when start_frame is given, else text-to-video.""" + if start_frame is not None: + validate_image_dimensions(start_frame, min_width=300, min_height=300) + validate_image_aspect_ratio(start_frame, (1, 2.5), (2.5, 1)) + contents = [Kling3TurboContent(type="first_frame", url=tensor_to_base64_string(start_frame))] + if prompt: + contents.insert(0, Kling3TurboContent(type="prompt", text=prompt)) + create = await sync_op( + cls, + ApiEndpoint(path="/proxy/kling/image-to-video/kling-3.0-turbo", method="POST"), + response_model=Kling3TurboCreateResponse, + data=Kling3TurboImage2VideoRequest( + contents=contents, + settings=Kling3TurboSettings(resolution=resolution, duration=duration), # i2v: no aspect_ratio + ), + ) + else: + create = await sync_op( + cls, + ApiEndpoint(path="/proxy/kling/text-to-video/kling-3.0-turbo", method="POST"), + response_model=Kling3TurboCreateResponse, + data=Kling3TurboText2VideoRequest( + prompt=prompt, + settings=Kling3TurboSettings(resolution=resolution, aspect_ratio=aspect_ratio, duration=duration), + ), + ) + if not (create.data and create.data.id): + raise RuntimeError(f"Kling 3.0 Turbo create failed. Code: {create.code}, Message: {create.message}") + final_response = await poll_op( + cls, + ApiEndpoint(path="/proxy/kling/tasks", query_params={"task_ids": create.data.id}), + response_model=Kling3TurboQueryResponse, + status_extractor=lambda r: (r.data[0].status if r.data else None), + ) + return IO.NodeOutput(await download_url_to_video_output(_turbo_video_url(final_response))) + + class KlingVideoNode(IO.ComfyNode): @classmethod @@ -2884,7 +2951,11 @@ class KlingVideoNode(IO.ComfyNode): ], tooltip="Generate a series of video segments with individual prompts and durations.", ), - IO.Boolean.Input("generate_audio", default=True), + IO.Boolean.Input( + "generate_audio", + default=True, + tooltip="'kling-3.0-turbo' always generates native audio, so the audio toggle is ignored.", + ), IO.DynamicCombo.Input( "model", options=[ @@ -2899,6 +2970,17 @@ class KlingVideoNode(IO.ComfyNode): ), ], ), + IO.DynamicCombo.Option( + "kling-3.0-turbo", + [ + IO.Combo.Input("resolution", options=["1080p", "720p"], default="720p"), + IO.Combo.Input( + "aspect_ratio", + options=["16:9", "9:16", "1:1"], + tooltip="Ignored in image-to-video mode.", + ), + ], + ), ], tooltip="Model and generation settings.", ), @@ -2930,6 +3012,7 @@ class KlingVideoNode(IO.ComfyNode): price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends( widgets=[ + "model", "model.resolution", "generate_audio", "multi_shot", @@ -2944,14 +3027,7 @@ class KlingVideoNode(IO.ComfyNode): ), expr=""" ( - $rates := { - "4k": {"off": 0.42, "on": 0.42}, - "1080p": {"off": 0.112, "on": 0.168}, - "720p": {"off": 0.084, "on": 0.126} - }; $res := $lookup(widgets, "model.resolution"); - $audio := widgets.generate_audio ? "on" : "off"; - $rate := $lookup($lookup($rates, $res), $audio); $ms := widgets.multi_shot; $isSb := $ms != "disabled"; $n := $isSb ? $number($substring($ms, 0, 1)) : 0; @@ -2962,7 +3038,18 @@ class KlingVideoNode(IO.ComfyNode): $d5 := $n >= 5 ? $lookup(widgets, "multi_shot.storyboard_5_duration") : 0; $d6 := $n >= 6 ? $lookup(widgets, "multi_shot.storyboard_6_duration") : 0; $dur := $isSb ? $d1 + $d2 + $d3 + $d4 + $d5 + $d6 : $lookup(widgets, "multi_shot.duration"); - {"type":"usd","usd": $rate * $dur} + widgets.model = "kling-3.0-turbo" + ? {"type":"usd","usd": ($res = "1080p" ? 0.14 : 0.112) * $dur} + : ( + $rates := { + "4k": {"off": 0.42, "on": 0.42}, + "1080p": {"off": 0.112, "on": 0.168}, + "720p": {"off": 0.084, "on": 0.126} + }; + $audio := widgets.generate_audio ? "on" : "off"; + $rate := $lookup($lookup($rates, $res), $audio); + {"type":"usd","usd": $rate * $dur} + ) ) """, ), @@ -3015,6 +3102,17 @@ class KlingVideoNode(IO.ComfyNode): duration = multi_shot["duration"] validate_string(multi_shot["prompt"], min_length=1, max_length=2500) + if model["model"] == "kling-3.0-turbo": + turbo_prompt = build_turbo_shot_prompt(multi_prompt_list) if custom_multi_shot else multi_shot["prompt"] + return await execute_kling_turbo( + cls, + prompt=turbo_prompt, + resolution=model["resolution"], + aspect_ratio=model["aspect_ratio"], + duration=duration, + start_frame=start_frame, + ) + if start_frame is not None: validate_image_dimensions(start_frame, min_width=300, min_height=300) validate_image_aspect_ratio(start_frame, (1, 2.5), (2.5, 1)) diff --git a/comfy_api_nodes/nodes_krea.py b/comfy_api_nodes/nodes_krea.py index 34369f05f..b9e6268f2 100644 --- a/comfy_api_nodes/nodes_krea.py +++ b/comfy_api_nodes/nodes_krea.py @@ -42,9 +42,11 @@ async def _upload_image_to_krea_assets(cls: type[IO.ComfyNode], image: Input.Ima _MODEL_MEDIUM = "Krea 2 Medium" +_MODEL_MEDIUM_TURBO = "Krea 2 Medium Turbo" _MODEL_LARGE = "Krea 2 Large" _MODEL_ENDPOINTS: dict[str, str] = { _MODEL_MEDIUM: "/proxy/krea/generate/image/krea/krea-2/medium", + _MODEL_MEDIUM_TURBO: "/proxy/krea/generate/image/krea/krea-2/medium-turbo", _MODEL_LARGE: "/proxy/krea/generate/image/krea/krea-2/large", } @@ -57,7 +59,7 @@ _UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F def _krea_model_inputs() -> list: - """Nested inputs shared by both Krea 2 Medium and Large under the DynamicCombo.""" + """Nested inputs shared by Krea 2 Medium, Medium Turbo and Large under the DynamicCombo.""" return [ IO.Combo.Input( "aspect_ratio", @@ -123,6 +125,7 @@ class Krea2ImageNode(IO.ComfyNode): "model", options=[ IO.DynamicCombo.Option(_MODEL_MEDIUM, _krea_model_inputs()), + IO.DynamicCombo.Option(_MODEL_MEDIUM_TURBO, _krea_model_inputs()), IO.DynamicCombo.Option(_MODEL_LARGE, _krea_model_inputs()), ], tooltip="Krea 2 Medium is best for expressive illustrations; " @@ -151,14 +154,15 @@ class Krea2ImageNode(IO.ComfyNode): ), expr=""" ( - $isLarge := widgets.model = "krea 2 large"; + $rates := { + "krea 2 medium turbo": {"text": 0.015, "style": 0.0175, "moodboard": 0.02}, + "krea 2 medium": {"text": 0.03, "style": 0.035, "moodboard": 0.04}, + "krea 2 large": {"text": 0.06, "style": 0.065, "moodboard": 0.07} + }; + $r := $lookup($rates, widgets.model); $hasMoodboard := $length($lookup(widgets, "model.moodboard_id")) > 0; $hasStyle := $lookup(inputs, "model.style_reference").connected; - $usd := $hasMoodboard - ? ($isLarge ? 0.07 : 0.04) - : ($hasStyle - ? ($isLarge ? 0.065 : 0.035) - : ($isLarge ? 0.06 : 0.03)); + $usd := $hasMoodboard ? $r.moodboard : ($hasStyle ? $r.style : $r.text); {"type":"usd","usd": $usd} ) """, diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 0d31ac77e..cdfa32d8b 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -3,9 +3,13 @@ from typing_extensions import override from comfy_api.latest import IO, ComfyExtension, Input from comfy_api_nodes.apis.luma import ( + LUMA_KEYFRAME_MODE_FRACTION, + LUMA_KEYFRAME_MODE_SECONDS, Luma2Generation, Luma2GenerationRequest, Luma2ImageRef, + Luma2VideoEdit, + Luma2VideoOptions, LumaAspectRatio, LumaCharacterRef, LumaConceptChain, @@ -18,6 +22,8 @@ from comfy_api_nodes.apis.luma import ( LumaIO, LumaKeyframes, LumaModifyImageRef, + LumaRay32KeyframeChain, + LumaRay32KeyframeItem, LumaReference, LumaReferenceChain, LumaVideoModel, @@ -33,6 +39,7 @@ from comfy_api_nodes.util import ( sync_op, upload_image_to_comfyapi, upload_images_to_comfyapi, + upload_video_to_comfyapi, validate_string, ) @@ -692,7 +699,10 @@ async def _luma2_upload_image_refs( async def _luma2_submit_and_poll( cls: type[IO.ComfyNode], request: Luma2GenerationRequest, -) -> Input.Image: + *, + estimated_duration: int | None = None, +) -> Luma2Generation: + """Submit a Luma Agents generation and poll until done; returns the completed generation.""" initial = await sync_op( cls, ApiEndpoint(path="/proxy/luma_2/generations", method="POST"), @@ -700,21 +710,21 @@ async def _luma2_submit_and_poll( data=request, ) if not initial.id: - raise RuntimeError("Luma 2 API did not return a generation id.") + raise RuntimeError("Luma API did not return a generation id.") final = await poll_op( cls, ApiEndpoint(path=f"/proxy/luma_2/generations/{initial.id}", method="GET"), response_model=Luma2Generation, status_extractor=lambda r: r.state, progress_extractor=lambda r: None, + estimated_duration=estimated_duration, ) - if not final.output: + if not final.output or not final.output[0].url: msg = final.failure_reason or "no output returned" - raise RuntimeError(f"Luma 2 generation failed: {msg}") - url = final.output[0].url - if not url: - raise RuntimeError("Luma 2 generation completed without an output URL.") - return await download_url_to_image_tensor(url) + if final.failure_code: + msg = f"{msg} [{final.failure_code}]" + raise RuntimeError(f"Luma generation failed: {msg}") + return final class LumaImageNode(IO.ComfyNode): @@ -843,7 +853,8 @@ class LumaImageNode(IO.ComfyNode): web_search=model["web_search"], image_ref=await _luma2_upload_image_refs(cls, model.get("image_ref"), max_count=9), ) - return IO.NodeOutput(await _luma2_submit_and_poll(cls, request)) + final = await _luma2_submit_and_poll(cls, request) + return IO.NodeOutput(await download_url_to_image_tensor(final.output[0].url)) class LumaImageEditNode(IO.ComfyNode): @@ -929,7 +940,533 @@ class LumaImageEditNode(IO.ComfyNode): web_search=model["web_search"], image_ref=await _luma2_upload_image_refs(cls, model.get("image_ref"), max_count=8), ) - return IO.NodeOutput(await _luma2_submit_and_poll(cls, request)) + final = await _luma2_submit_and_poll(cls, request) + return IO.NodeOutput(await download_url_to_image_tensor(final.output[0].url)) + + +_BADGE_RAY32_VIDEO = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution", "duration"]), + expr=""" + ( + $p := { + "360p": {"5s": 0.06, "10s": 0.18}, + "540p": {"5s": 0.15, "10s": 0.45}, + "720p": {"5s": 0.3, "10s": 0.9}, + "1080p": {"5s": 1.2, "10s": 3.6} + }; + {"type": "usd", "usd": $lookup($lookup($p, widgets.resolution), widgets.duration)} + ) + """, +) + +_BADGE_RAY32_VIDEO_5S = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + ( + $p := {"360p": 0.06, "540p": 0.15, "720p": 0.3, "1080p": 1.2}; + {"type": "usd", "usd": $lookup($p, widgets.resolution)} + ) + """, +) + +_BADGE_RAY32_EDIT = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + ( + $p := { + "360p": {"min": 0.54, "max": 1.08}, + "540p": {"min": 0.72, "max": 1.44}, + "720p": {"min": 1.08, "max": 2.16}, + "1080p": {"min": 2.16, "max": 4.32} + }; + $r := $lookup($p, widgets.resolution); + {"type": "range_usd", "min_usd": $r.min, "max_usd": $r.max, "format": {"note": "(by source length)"}} + ) + """, +) + +_BADGE_RAY32_REFRAME = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + ( + $p := {"360p": 0.03, "540p": 0.06, "720p": 0.12, "1080p": 0.36}; + {"type": "usd", "usd": $lookup($p, widgets.resolution), "format": {"suffix": "/second"}} + ) + """, +) + + +def _ray32_seed_input() -> IO.Input: + return IO.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + control_after_generate=True, + tooltip="Seed to determine if node should re-run; results are nondeterministic regardless of seed.", + ) + + +async def _ray32_generate(cls: type[IO.ComfyNode], request: Luma2GenerationRequest) -> IO.NodeOutput: + """Run a ray-3.2 generation and return (video, generation_id).""" + final = await _luma2_submit_and_poll(cls, request, estimated_duration=120) + video = await download_url_to_video_output(final.output[0].url) + return IO.NodeOutput(video, final.id or "") + + +class LumaRay32TextToVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32TextToVideoNode", + display_name="Luma Ray 3.2 Text to Video", + category="partner/video/Luma", + description="Generate a video from a text prompt using Luma's Ray 3.2 model.", + inputs=[ + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the video generation."), + IO.Combo.Input("aspect_ratio", options=["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Combo.Input("duration", options=["5s", "10s"]), + IO.Boolean.Input( + "loop", + default=False, + tooltip="Make the video loop seamlessly. Only available with 5s duration.", + ), + _ray32_seed_input(), + ], + outputs=[IO.Video.Output(), IO.String.Output(display_name="generation_id")], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO, + ) + + @classmethod + async def execute( + cls, prompt: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + if loop and duration == "10s": + raise ValueError("Looping is only available with 5s duration on Ray 3.2.") + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video", + aspect_ratio=aspect_ratio, + video=Luma2VideoOptions(resolution=resolution, duration=duration, loop=loop or None), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32ImageToVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32ImageToVideoNode", + display_name="Luma Ray 3.2 Image to Video", + category="partner/video/Luma", + description="Generate a video from a start and/or end frame using Luma's Ray 3.2 model. " + "Image-anchored generations are always 5 seconds.", + inputs=[ + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the video generation."), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Boolean.Input( + "loop", + default=False, + tooltip="Make the video loop seamlessly. Not available when an end_frame is set.", + ), + _ray32_seed_input(), + IO.Image.Input("start_frame", optional=True, tooltip="First frame of the generated video."), + IO.Image.Input("end_frame", optional=True, tooltip="Last frame of the generated video."), + ], + outputs=[IO.Video.Output(), IO.String.Output(display_name="generation_id")], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO_5S, + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + loop: bool, + seed: int, + start_frame: torch.Tensor | None = None, + end_frame: torch.Tensor | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + if start_frame is None and end_frame is None: + raise ValueError("Provide at least one of start_frame / end_frame.") + if loop and end_frame is not None: + raise ValueError("Looping is not available when an end_frame is set.") + video = Luma2VideoOptions(resolution=resolution, duration="5s", loop=loop or None) + if start_frame is not None: + url = await upload_image_to_comfyapi(cls, start_frame, mime_type="image/png") + video.start_frame = Luma2ImageRef(url=url) + if end_frame is not None: + url = await upload_image_to_comfyapi(cls, end_frame, mime_type="image/png") + video.end_frame = Luma2ImageRef(url=url) + request = Luma2GenerationRequest(prompt=prompt, model="ray-3.2", type="video", video=video) + return await _ray32_generate(cls, request) + + +class LumaRay32KeyframeNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32KeyframeNode", + display_name="Luma Ray 3.2 Keyframe", + category="partner/video/Luma", + description="Anchor a guide image to a position on the Ray 3.2 output video timeline. Connect this to " + "the 'keyframes' input of the Luma Ray 3.2 Keyframes to Video node; chain several together via the " + "optional 'keyframes' input below.", + inputs=[ + IO.Image.Input("image", tooltip="Guide image to place at the chosen moment of the output video."), + IO.DynamicCombo.Input( + "position", + options=[ + IO.DynamicCombo.Option( + "Fraction of duration (0.0-1.0)", + [ + IO.Float.Input( + "fraction", + default=0.0, + min=0.0, + max=1.0, + step=0.01, + display_mode=IO.NumberDisplay.number, + tooltip="Where in the output video this image applies " "(0.0 = start, 1.0 = end).", + ), + ], + ), + IO.DynamicCombo.Option( + "Absolute time (seconds)", + [ + IO.Float.Input( + "seconds", + default=0.0, + min=0.0, + max=10.0, + step=0.1, + display_mode=IO.NumberDisplay.number, + tooltip="Time in seconds from the start of the output video where this " + "image applies.", + ), + ], + ), + ], + tooltip="How to place this image on the output video's timeline.", + ), + IO.Custom(LumaIO.LUMA_RAY32_KEYFRAME).Input( + "keyframes", + optional=True, + tooltip="Optional earlier keyframes to chain with this one.", + ), + ], + outputs=[IO.Custom(LumaIO.LUMA_RAY32_KEYFRAME).Output(display_name="keyframes")], + ) + + @classmethod + def execute( + cls, + image: torch.Tensor, + position: dict, + keyframes: LumaRay32KeyframeChain | None = None, + ) -> IO.NodeOutput: + chain = keyframes.clone() if keyframes is not None else LumaRay32KeyframeChain() + if position["position"] == "Absolute time (seconds)": + mode, value = LUMA_KEYFRAME_MODE_SECONDS, float(position["seconds"]) + else: + mode, value = LUMA_KEYFRAME_MODE_FRACTION, float(position["fraction"]) + chain.add(LumaRay32KeyframeItem(image=image, mode=mode, value=value)) + return IO.NodeOutput(chain) + + +class LumaRay32KeyframesToVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32KeyframesToVideoNode", + display_name="Luma Ray 3.2 Keyframes to Video", + category="partner/video/Luma", + description="Generate a video that interpolates through a sequence of guide images, each anchored to a " + "position on the timeline, using Luma Ray 3.2. Build the sequence with Luma Ray 3.2 Keyframe nodes " + "(at least 2).", + inputs=[ + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the video generation."), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Combo.Input("duration", options=["5s", "10s"]), + _ray32_seed_input(), + IO.Custom(LumaIO.LUMA_RAY32_KEYFRAME).Input( + "keyframes", + tooltip="Keyframe sequence from Luma Ray 3.2 Keyframe nodes (at least 2).", + ), + ], + outputs=[IO.Video.Output(), IO.String.Output(display_name="generation_id")], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO, + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + duration: str, + seed: int, + keyframes: LumaRay32KeyframeChain | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + items = keyframes.items if keyframes is not None else [] + if len(items) < 2: + raise ValueError( + "Connect at least 2 Luma Ray 3.2 Keyframe nodes " + "(use Luma Ray 3.2 Image to Video for a single start/end frame)." + ) + if len(items) > 64: + raise ValueError(f"Ray 3.2 supports at most 64 keyframes; got {len(items)}.") + maxframe = 120 if duration == "5s" else 240 + duration_seconds = maxframe / 24 # 5.0 or 10.0 + # Resolve each keyframe to an output-frame index, then order by position + # (so the user can chain keyframes in any order — the position is what places them) + placed: list[tuple[int, torch.Tensor]] = [] + for item in items: + if item.mode == LUMA_KEYFRAME_MODE_SECONDS: + if item.value > duration_seconds: + raise ValueError( + f"Keyframe position {item.value:g}s is past the end of the {duration} video; " + f"use 0-{duration_seconds:g}s (or switch the keyframe to fraction mode)." + ) + idx = round(item.value * 24) + else: + idx = round(item.value * maxframe) + placed.append((max(0, min(maxframe, idx)), item.image)) + placed.sort(key=lambda p: p[0]) + indexes = [idx for idx, _ in placed] + for a, b in zip(indexes, indexes[1:]): + if a == b: + raise ValueError( + f"Two keyframes resolve to the same output frame ({a}) for a {duration} video " + f"(valid range 0-{maxframe}); give each keyframe a distinct position." + ) + refs: list[Luma2ImageRef] = [] + for _, image in placed: + url = await upload_image_to_comfyapi(cls, image, mime_type="image/png") + refs.append(Luma2ImageRef(url=url)) + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video", + video=Luma2VideoOptions(resolution=resolution, duration=duration, keyframes=refs, keyframe_indexes=indexes), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32VideoEditNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32VideoEditNode", + display_name="Luma Ray 3.2 Video Edit", + category="partner/video/Luma", + description="Re-render an existing video under a new prompt using Luma Ray 3.2 (restyle, relight, add " + "or remove elements) while keeping the original motion. Source video up to 18 seconds; the edited " + "video keeps the source's length.", + inputs=[ + IO.Video.Input("video", tooltip="Source video to edit. Up to 18 seconds."), + IO.String.Input("prompt", multiline=True, default="", tooltip="Describes the desired edit."), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Combo.Input( + "strength", + options=[ + "auto", + "adhere_1", + "adhere_2", + "adhere_3", + "flex_1", + "flex_2", + "flex_3", + "reimagine_1", + "reimagine_2", + "reimagine_3", + ], + default="auto", + tooltip="How strongly to preserve vs. reimagine the source. 'auto' lets Ray 3.2 choose; " + "adhere_* preserves the most, flex_* is balanced, reimagine_* changes the most.", + ), + _ray32_seed_input(), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(display_name="generation_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_EDIT, + ) + + @classmethod + async def execute( + cls, video: Input.Video, prompt: str, resolution: str, strength: str, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + try: + duration = "5s" if video.get_duration() <= 5.0 else "10s" + except Exception: + duration = "10s" + source_url = await upload_video_to_comfyapi(cls, video, max_duration=18) + edit = Luma2VideoEdit(auto_controls=True) if strength == "auto" else Luma2VideoEdit(strength=strength) + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video_edit", + source=Luma2ImageRef(url=source_url, media_type="video/mp4"), + video=Luma2VideoOptions(resolution=resolution, duration=duration, edit=edit), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32VideoReframeNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32VideoReframeNode", + display_name="Luma Ray 3.2 Video Reframe", + category="partner/video/Luma", + description="Change the aspect ratio of an existing video, using Luma Ray 3.2 to fill the newly " + "exposed canvas areas. Source video up to 30 seconds. Billed per second of output.", + inputs=[ + IO.Video.Input("video", tooltip="Source video to reframe. Up to 30 seconds."), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describes how the newly exposed canvas areas should be filled.", + ), + IO.Combo.Input("aspect_ratio", options=["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + _ray32_seed_input(), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(display_name="generation_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_REFRAME, + ) + + @classmethod + async def execute( + cls, video: Input.Video, prompt: str, aspect_ratio: str, resolution: str, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=False, min_length=1, max_length=6000) + if resolution == "1080p" and aspect_ratio in {"9:16", "3:4"}: + raise ValueError("1080p is not available for vertical aspect ratios (9:16, 3:4) when reframing.") + source_url = await upload_video_to_comfyapi(cls, video, max_duration=30) + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video_reframe", + aspect_ratio=aspect_ratio, + source=Luma2ImageRef(url=source_url, media_type="video/mp4"), + video=Luma2VideoOptions(resolution=resolution), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32ExtendVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32ExtendVideoNode", + display_name="Luma Ray 3.2 Extend Video", + category="partner/video/Luma", + description="Extend a previous Ray 3.2 generation forward (continue after it) or backward (lead-in " + "before it). Connect the generation_id output of a prior Luma Ray 3.2 node." + " Extensions are always 5 seconds.", + inputs=[ + IO.String.Input( + "source_generation_id", + default="", + tooltip="generation_id of the prior Ray 3.2 video to extend." + " Connect the generation_id output of another Luma Ray 3.2 node.", + ), + IO.DynamicCombo.Input( + "direction", + options=[ + IO.DynamicCombo.Option( + "Forward (continue after)", + [ + IO.Boolean.Input( + "loop", + default=False, + tooltip="Loop the extended video seamlessly (forward extend only).", + ), + ], + ), + IO.DynamicCombo.Option("Backward (lead-in before)", []), + ], + tooltip="Forward continues after the prior clip; backward is prepended before it.", + ), + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the new content."), + IO.Combo.Input("resolution", options=["540p", "720p", "1080p"], default="720p"), + _ray32_seed_input(), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(display_name="generation_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO_5S, + ) + + @classmethod + async def execute( + cls, source_generation_id: str, direction: dict, prompt: str, resolution: str, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=False, min_length=1, max_length=6000) + gen_id = (source_generation_id or "").strip() + if not gen_id: + raise ValueError( + "source_generation_id is required (connect the generation_id output of a prior Luma Ray 3.2 node)." + ) + video = Luma2VideoOptions(resolution=resolution, duration="5s") + ref = Luma2ImageRef(generation_id=gen_id) + if direction["direction"] == "Forward (continue after)": + video.start_frame = ref + if direction.get("loop"): + video.loop = True + else: + video.end_frame = ref + request = Luma2GenerationRequest(prompt=prompt, model="ray-3.2", type="video", video=video) + return await _ray32_generate(cls, request) class LumaExtension(ComfyExtension): @@ -944,6 +1481,13 @@ class LumaExtension(ComfyExtension): LumaConceptsNode, LumaImageNode, LumaImageEditNode, + LumaRay32TextToVideoNode, + LumaRay32ImageToVideoNode, + LumaRay32KeyframeNode, + LumaRay32KeyframesToVideoNode, + LumaRay32VideoEditNode, + LumaRay32VideoReframeNode, + LumaRay32ExtendVideoNode, ] diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 0fe5fb9d0..ad62f2164 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -9,6 +9,7 @@ from PIL import Image from typing_extensions import override import folder_paths +from comfy.utils import common_upscale from comfy_api.latest import IO, ComfyExtension, Input from comfy_api_nodes.apis.openai import ( InputFileContent, @@ -62,7 +63,8 @@ async def validate_and_cast_response(response, timeout: int = None) -> torch.Ten timeout: Request timeout in seconds. Defaults to None (no timeout). Returns: - A torch.Tensor representing the image (1, H, W, C). + A torch.Tensor of shape (N, H, W, C) with all returned images; images whose + dimensions differ from the first image's are resized to match it. Raises: ValueError: If the response is not valid. @@ -89,6 +91,14 @@ async def validate_and_cast_response(response, timeout: int = None) -> torch.Ten arr = np.asarray(pil_img).astype(np.float32) / 255.0 image_tensors.append(torch.from_numpy(arr)) + # With size="auto" the API can return images whose dimensions differ by a few pixels within a single response + # resize them to the first image's dimensions so they can be stacked into one batch. + ref_h, ref_w = image_tensors[0].shape[:2] + for i, t in enumerate(image_tensors): + if t.shape[:2] != (ref_h, ref_w): + samples = t.unsqueeze(0).movedim(-1, 1) + samples = common_upscale(samples, ref_w, ref_h, "bilinear", "center") + image_tensors[i] = samples.movedim(1, -1).squeeze(0) return torch.stack(image_tensors, dim=0) diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index b9c5c81a1..013a193d9 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -30,13 +30,33 @@ from comfy_api_nodes.apis.runway import ( Model4, ReferenceImage, RunwayTextToImageAspectRatioEnum, + RunwayAleph2IO, + RunwayAleph2KeyframeChain, + RunwayAleph2KeyframeItem, + RunwayAleph2PromptImageChain, + RunwayAleph2PromptImageItem, + RunwayAleph2Request, + RunwayAleph2Response, + RunwayAleph2KeyframeSeconds, + RunwayAleph2KeyframeAt, + RunwayAleph2PromptImage, + RunwayAleph2TimestampPosition, + RunwayAleph2RelativePosition, + RunwayAleph2ContentModeration, + KEYFRAME_MODE_SECONDS, + KEYFRAME_MODE_AT, + PROMPT_IMAGE_MODE_TIMESTAMP, + PROMPT_IMAGE_MODE_POSITION, ) from comfy_api_nodes.util import ( image_tensor_pair_to_batch, validate_string, validate_image_dimensions, validate_image_aspect_ratio, + validate_video_duration, upload_images_to_comfyapi, + upload_image_to_comfyapi, + upload_video_to_comfyapi, download_url_to_video_output, download_url_to_image_tensor, ApiEndpoint, @@ -45,6 +65,7 @@ from comfy_api_nodes.util import ( ) PATH_IMAGE_TO_VIDEO = "/proxy/runway/image_to_video" +PATH_VIDEO_TO_VIDEO = "/proxy/runway/video_to_video" PATH_TEXT_TO_IMAGE = "/proxy/runway/text_to_image" PATH_GET_TASK_STATUS = "/proxy/runway/tasks" @@ -53,12 +74,6 @@ AVERAGE_DURATION_FLF_SECONDS = 256 AVERAGE_DURATION_T2I_SECONDS = 41 -class RunwayApiError(Exception): - """Base exception for Runway API errors.""" - - pass - - class RunwayGen4TurboAspectRatio(str, Enum): """Aspect ratios supported for Image to Video API when using gen4_turbo model.""" @@ -84,14 +99,6 @@ def get_video_url_from_task_status(response: TaskStatusResponse) -> str | None: return None -def extract_progress_from_task_status( - response: TaskStatusResponse, -) -> float | None: - if hasattr(response, "progress") and response.progress is not None: - return response.progress * 100 - return None - - def get_image_url_from_task_status(response: TaskStatusResponse) -> str | None: """Returns the image URL from the task status response if it exists.""" if hasattr(response, "output") and len(response.output) > 0: @@ -102,14 +109,13 @@ def get_image_url_from_task_status(response: TaskStatusResponse) -> str | None: async def get_response( cls: type[IO.ComfyNode], task_id: str, estimated_duration: int | None = None ) -> TaskStatusResponse: - """Poll the task status until it is finished then get the response.""" return await poll_op( cls, ApiEndpoint(path=f"{PATH_GET_TASK_STATUS}/{task_id}"), response_model=TaskStatusResponse, - status_extractor=lambda r: r.status.value, + status_extractor=lambda r: r.status, estimated_duration=estimated_duration, - progress_extractor=extract_progress_from_task_status, + progress_extractor=lambda r: r.progress * 100 if r.progress is not None else None, ) @@ -127,7 +133,7 @@ async def generate_video( final_response = await get_response(cls, initial_response.id, estimated_duration) if not final_response.output: - raise RunwayApiError("Runway task succeeded but no video data found in response.") + raise ValueError("Runway task succeeded but no video data found in response.") video_url = get_video_url_from_task_status(final_response) return await download_url_to_video_output(video_url) @@ -410,7 +416,7 @@ class RunwayFirstLastFrameNode(IO.ComfyNode): mime_type="image/png", ) if len(download_urls) != 2: - raise RunwayApiError("Failed to upload one or more images to comfy api.") + raise ValueError("Failed to upload one or more images to comfy api.") return IO.NodeOutput( await generate_video( @@ -514,11 +520,321 @@ class RunwayTextToImageNode(IO.ComfyNode): estimated_duration=AVERAGE_DURATION_T2I_SECONDS, ) if not final_response.output: - raise RunwayApiError("Runway task succeeded but no image data found in response.") + raise ValueError("Runway task succeeded but no image data found in response.") return IO.NodeOutput(await download_url_to_image_tensor(get_image_url_from_task_status(final_response))) +_TIMING_ABSOLUTE = "Absolute time (seconds)" +_TIMING_FRACTION = "Fraction of duration (0.0-1.0)" + + +class RunwayAleph2KeyframeNode(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RunwayAleph2KeyframeNode", + display_name="Runway Aleph2 Keyframe", + category="partner/video/Runway", + description="Anchor a guidance image to a moment of the input (source) video, so Aleph2 " + "steers the edit at that point of your footage. Connect this to the 'keyframes' input of " + "the Runway Aleph2 Video to Video node; chain several together (up to 5) via the optional " + "'keyframes' input below.", + inputs=[ + IO.Image.Input( + "image", + tooltip="The guidance image to apply at the chosen moment of the input video.", + ), + IO.DynamicCombo.Input( + "timing", + options=[ + IO.DynamicCombo.Option( + _TIMING_ABSOLUTE, + [ + IO.Float.Input( + "seconds", + default=0.0, + min=0.0, + max=30.0, + step=0.1, + display_mode=IO.NumberDisplay.number, + tooltip="Time in seconds from start of the input video where this image applies.", + ), + ], + ), + IO.DynamicCombo.Option( + _TIMING_FRACTION, + [ + IO.Float.Input( + "fraction", + default=0.0, + min=0.0, + max=1.0, + step=0.01, + display_mode=IO.NumberDisplay.number, + tooltip="Where in the input video this image applies, " + "as a fraction of its duration (0.0 = start, 1.0 = end).", + ), + ], + ), + ], + tooltip="How to place this image on the input video's timeline.", + ), + IO.Custom(RunwayAleph2IO.KEYFRAME).Input( + "keyframes", + optional=True, + tooltip="Optional earlier keyframes to chain with this one.", + ), + ], + outputs=[IO.Custom(RunwayAleph2IO.KEYFRAME).Output(display_name="keyframes")], + ) + + @classmethod + def execute( + cls, + image: Input.Image, + timing: dict, + keyframes: RunwayAleph2KeyframeChain | None = None, + ) -> IO.NodeOutput: + chain = keyframes.clone() if keyframes is not None else RunwayAleph2KeyframeChain() + if timing["timing"] == _TIMING_ABSOLUTE: + mode, value = KEYFRAME_MODE_SECONDS, float(timing["seconds"]) + else: + mode, value = KEYFRAME_MODE_AT, float(timing["fraction"]) + chain.add(RunwayAleph2KeyframeItem(image=image, mode=mode, value=value)) + return IO.NodeOutput(chain) + + +class RunwayAleph2PromptImageNode(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RunwayAleph2PromptImageNode", + display_name="Runway Aleph2 Prompt Image", + category="partner/video/Runway", + description="Anchor a guidance image to a moment of the output (result) video, to guide what " + "the edited video looks like at that point. Connect this to the 'prompt_images' input of the " + "Runway Aleph2 Video to Video node; chain several together (up to 5) via the optional " + "'prompt_images' input below.", + inputs=[ + IO.Image.Input( + "image", + tooltip="The guidance image to place at the chosen moment of the output video.", + ), + IO.DynamicCombo.Input( + "position", + options=[ + IO.DynamicCombo.Option( + _TIMING_ABSOLUTE, + [ + IO.Float.Input( + "seconds", + default=0.0, + min=0.0, + max=30.0, + step=0.1, + display_mode=IO.NumberDisplay.number, + tooltip="Time in seconds from start of the output video where this image applies.", + ), + ], + ), + IO.DynamicCombo.Option( + _TIMING_FRACTION, + [ + IO.Float.Input( + "fraction", + default=0.0, + min=0.0, + max=1.0, + step=0.01, + display_mode=IO.NumberDisplay.number, + tooltip="Where in the output video this image applies, " + "as a fraction of its duration (0.0 = start, 1.0 = end).", + ), + ], + ), + ], + tooltip="How to place this image on the output video's timeline.", + ), + IO.Custom(RunwayAleph2IO.PROMPT_IMAGE).Input( + "prompt_images", + optional=True, + tooltip="Optional earlier prompt images to chain with this one.", + ), + ], + outputs=[IO.Custom(RunwayAleph2IO.PROMPT_IMAGE).Output(display_name="prompt_images")], + ) + + @classmethod + def execute( + cls, + image: Input.Image, + position: dict, + prompt_images: RunwayAleph2PromptImageChain | None = None, + ) -> IO.NodeOutput: + chain = prompt_images.clone() if prompt_images is not None else RunwayAleph2PromptImageChain() + if position["position"] == _TIMING_ABSOLUTE: + mode, value = PROMPT_IMAGE_MODE_TIMESTAMP, float(position["seconds"]) + else: + mode, value = PROMPT_IMAGE_MODE_POSITION, float(position["fraction"]) + chain.add(RunwayAleph2PromptImageItem(image=image, mode=mode, value=value)) + return IO.NodeOutput(chain) + + +class RunwayAleph2VideoToVideoNode(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RunwayAleph2VideoToVideoNode", + display_name="Runway Aleph2 Video to Video", + category="partner/video/Runway", + description="Edit a video with a text prompt using Runway's Aleph2 model. Aleph2 transforms " + "your footage (restyle, relight, add or remove elements, change the viewpoint) while keeping " + "the original motion and timing; the output resolution matches the input video, which must be " + "2-30 seconds at 30 fps or lower. Optionally steer the edit with either keyframes (anchored to " + "the input video) or prompt images (anchored to the output video) - use one or the other, not both.", + inputs=[ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describes what should appear in the output (1-1000 characters).", + ), + IO.Video.Input( + "video", + tooltip="Input video to edit. Must be 2-30 seconds at 30 fps or lower.", + ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=4294967295, + step=1, + control_after_generate=True, + display_mode=IO.NumberDisplay.number, + tooltip="Random seed for generation", + ), + IO.Combo.Input( + "public_figure_threshold", + options=["auto", "low"], + default="low", + tooltip="Content moderation for recognizable public figures.", + ), + IO.Custom(RunwayAleph2IO.KEYFRAME).Input( + "keyframes", + optional=True, + tooltip="Guidance images anchored to the input video, from Aleph2 Keyframe nodes (up to 5). " + "Use keyframes or prompt images, not both.", + ), + IO.Custom(RunwayAleph2IO.PROMPT_IMAGE).Input( + "prompt_images", + optional=True, + tooltip="Guidance images anchored to the output video, from Aleph2 Prompt Image nodes (up to 5). " + "Use keyframes or prompt images, not both.", + ), + ], + outputs=[ + IO.Video.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd": 0.4004, "format":{"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + prompt: str, + video: Input.Video, + seed: int, + public_figure_threshold: str = "low", + keyframes: RunwayAleph2KeyframeChain | None = None, + prompt_images: RunwayAleph2PromptImageChain | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, min_length=1, max_length=1000) + validate_video_duration( + video, + min_duration=2.0, + max_duration=30.0, + ) + try: + fps = float(video.get_frame_rate()) + except Exception: + fps = None + if fps is not None and fps > 30.0 + 0.01: + raise ValueError(f"Input video frame rate ({fps:.2f} fps) exceeds Aleph2's maximum of 30 fps.") + + if (keyframes and keyframes.items) and (prompt_images and prompt_images.items): + raise ValueError("Aleph2 accepts either keyframes or prompt images, not both.") + + video_duration: float | None = None + try: + video_duration = video.get_duration() + except Exception: + video_duration = None + + def _check_seconds(value: float, label: str) -> None: + if video_duration is not None and value > video_duration + 0.0001: + raise ValueError(f"{label} {value:.2f}s exceeds the input video duration ({video_duration:.2f}s).") + + video_url = await upload_video_to_comfyapi(cls, video) + + keyframe_models: list[RunwayAleph2KeyframeSeconds | RunwayAleph2KeyframeAt] = [] + if keyframes is not None: + if len(keyframes.items) > 5: + raise ValueError("Aleph2 supports at most 5 keyframes.") + for item in keyframes.items: + image_url = await upload_image_to_comfyapi(cls, item.image, mime_type="image/png") + if item.mode == KEYFRAME_MODE_SECONDS: + _check_seconds(item.value, "Keyframe timestamp") + keyframe_models.append(RunwayAleph2KeyframeSeconds(seconds=item.value, uri=image_url)) + else: + keyframe_models.append(RunwayAleph2KeyframeAt(at=item.value, uri=image_url)) + + prompt_image_models: list[RunwayAleph2PromptImage] = [] + if prompt_images is not None: + if len(prompt_images.items) > 5: + raise ValueError("Aleph2 supports at most 5 prompt images.") + for item in prompt_images.items: + image_url = await upload_image_to_comfyapi(cls, item.image, mime_type="image/png") + position: RunwayAleph2TimestampPosition | RunwayAleph2RelativePosition + if item.mode == PROMPT_IMAGE_MODE_TIMESTAMP: + _check_seconds(item.value, "Prompt image timestamp") + position = RunwayAleph2TimestampPosition(timestampSeconds=item.value) + else: + position = RunwayAleph2RelativePosition(positionPercentage=item.value) + prompt_image_models.append(RunwayAleph2PromptImage(position=position, uri=image_url)) + + initial_response = await sync_op( + cls, + endpoint=ApiEndpoint(path=PATH_VIDEO_TO_VIDEO, method="POST"), + response_model=RunwayAleph2Response, + data=RunwayAleph2Request( + promptText=prompt, + videoUri=video_url, + seed=seed, + contentModeration=RunwayAleph2ContentModeration(publicFigureThreshold=public_figure_threshold), + keyframes=keyframe_models or None, + promptImage=prompt_image_models or None, + ), + ) + + final_response = await get_response(cls, initial_response.id) + if not final_response.output: + raise ValueError("Runway task succeeded but no video data found in response.") + + return IO.NodeOutput(await download_url_to_video_output(get_video_url_from_task_status(final_response))) + + class RunwayExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -527,6 +843,9 @@ class RunwayExtension(ComfyExtension): RunwayImageToVideoNodeGen3a, RunwayImageToVideoNodeGen4, RunwayTextToImageNode, + RunwayAleph2VideoToVideoNode, + RunwayAleph2KeyframeNode, + RunwayAleph2PromptImageNode, ] diff --git a/comfy_api_nodes/nodes_sonilo.py b/comfy_api_nodes/nodes_sonilo.py index 9ce896ed0..2ad35531a 100644 --- a/comfy_api_nodes/nodes_sonilo.py +++ b/comfy_api_nodes/nodes_sonilo.py @@ -16,7 +16,7 @@ from comfy_api_nodes.util import ( ) from comfy_api_nodes.util._helpers import ( default_base_url, - get_auth_header, + get_comfy_api_headers, get_node_id, is_processing_interrupted, ) @@ -100,8 +100,7 @@ class SoniloTextToMusic(IO.ComfyNode): node_id="SoniloTextToMusic", display_name="Sonilo Text to Music", category="partner/audio/Sonilo", - description="Generate music from a text prompt using Sonilo's AI model. " - "Leave duration at 0 to let the model infer it from the prompt.", + description="Generate music from a text prompt using Sonilo's AI model.", inputs=[ IO.String.Input( "prompt", @@ -111,11 +110,10 @@ class SoniloTextToMusic(IO.ComfyNode): ), IO.Int.Input( "duration", - default=0, - min=0, + default=30, + min=1, max=360, - tooltip="Target duration in seconds. Set to 0 to let the model " - "infer the duration from the prompt. Maximum: 6 minutes.", + tooltip="Target duration in seconds. Maximum: 6 minutes.", ), IO.Int.Input( "seed", @@ -136,13 +134,7 @@ class SoniloTextToMusic(IO.ComfyNode): is_api_node=True, price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends(widgets=["duration"]), - expr=""" - ( - widgets.duration > 0 - ? {"type":"usd","usd": 0.005 * widgets.duration} - : {"type":"usd","usd": 0.005, "format":{"suffix":"/second"}} - ) - """, + expr='{"type":"usd","usd": 0.0025 * widgets.duration}', ), ) @@ -150,14 +142,13 @@ class SoniloTextToMusic(IO.ComfyNode): async def execute( cls, prompt: str, - duration: int = 0, + duration: int = 1, seed: int = 0, ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=True, min_length=1) + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=1000) form = aiohttp.FormData() form.add_field("prompt", prompt) - if duration > 0: - form.add_field("duration", str(duration)) + form.add_field("duration", str(duration)) audio_bytes = await _stream_sonilo_music( cls, ApiEndpoint(path="/proxy/sonilo/t2m/generate", method="POST"), @@ -174,8 +165,7 @@ async def _stream_sonilo_music( """POST ``form`` to Sonilo, read the NDJSON stream, and return the first stream's audio bytes.""" url = urljoin(default_base_url().rstrip("/") + "/", endpoint.path.lstrip("/")) - headers: dict[str, str] = {} - headers.update(get_auth_header(cls)) + headers = get_comfy_api_headers(cls) headers.update(endpoint.headers) node_id = get_node_id(cls) diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py deleted file mode 100644 index 9eaba173b..000000000 --- a/comfy_api_nodes/nodes_stability.py +++ /dev/null @@ -1,932 +0,0 @@ -from inspect import cleandoc -from typing import Optional -from typing_extensions import override - -from comfy_api.latest import ComfyExtension, Input, IO -from comfy_api_nodes.apis.stability import ( - StabilityUpscaleConservativeRequest, - StabilityUpscaleCreativeRequest, - StabilityAsyncResponse, - StabilityResultsGetResponse, - StabilityStable3_5Request, - StabilityStableUltraRequest, - StabilityStableUltraResponse, - StabilityAspectRatio, - Stability_SD3_5_Model, - Stability_SD3_5_GenerationMode, - get_stability_style_presets, - StabilityTextToAudioRequest, - StabilityAudioToAudioRequest, - StabilityAudioInpaintRequest, - StabilityAudioResponse, -) -from comfy_api_nodes.util import ( - validate_audio_duration, - validate_string, - audio_input_to_mp3, - bytesio_to_image_tensor, - tensor_to_bytesio, - audio_bytes_to_audio_input, - sync_op, - poll_op, - ApiEndpoint, -) - -import torch -import base64 -from io import BytesIO -from enum import Enum - - -class StabilityPollStatus(str, Enum): - finished = "finished" - in_progress = "in_progress" - failed = "failed" - - -def get_async_dummy_status(x: StabilityResultsGetResponse): - if x.name is not None or x.errors is not None: - return StabilityPollStatus.failed - elif x.finish_reason is not None: - return StabilityPollStatus.finished - return StabilityPollStatus.in_progress - - -class StabilityStableImageUltraNode(IO.ComfyNode): - """ - Generates images synchronously based on prompt and resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityStableImageUltraNode", - display_name="Stability AI Stable Image Ultra", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines" + - "elements, colors, and subjects will lead to better results. " + - "To control the weight of a given word use the format `(word:weight)`," + - "where `word` is the word you'd like to control the weight of and `weight`" + - "is a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`" + - "would convey a sky that was blue and green, but more green than blue.", - ), - IO.Combo.Input( - "aspect_ratio", - options=StabilityAspectRatio, - default=StabilityAspectRatio.ratio_1_1, - tooltip="Aspect ratio of generated image.", - ), - IO.Combo.Input( - "style_preset", - options=get_stability_style_presets(), - tooltip="Optional desired style of generated image.", - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.Image.Input( - "image", - optional=True, - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="A blurb of text describing what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - IO.Float.Input( - "image_denoise", - default=0.5, - min=0.0, - max=1.0, - step=0.01, - tooltip="Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.", - optional=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.08}""", - ), - ) - - @classmethod - async def execute( - cls, - prompt: str, - aspect_ratio: str, - style_preset: str, - seed: int, - image: Optional[torch.Tensor] = None, - negative_prompt: str = "", - image_denoise: Optional[float] = 0.5, - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - # prepare image binary if image present - image_binary = None - if image is not None: - image_binary = tensor_to_bytesio(image, total_pixels=1504*1504).read() - else: - image_denoise = None - - if not negative_prompt: - negative_prompt = None - if style_preset == "None": - style_preset = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/generate/ultra", method="POST"), - response_model=StabilityStableUltraResponse, - data=StabilityStableUltraRequest( - prompt=prompt, - negative_prompt=negative_prompt, - aspect_ratio=aspect_ratio, - seed=seed, - strength=image_denoise, - style_preset=style_preset, - ), - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stable Image Ultra generation failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityStableImageSD_3_5Node(IO.ComfyNode): - """ - Generates images synchronously based on prompt and resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityStableImageSD_3_5Node", - display_name="Stability AI Stable Diffusion 3.5 Image", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", - ), - IO.Combo.Input( - "model", - options=Stability_SD3_5_Model, - ), - IO.Combo.Input( - "aspect_ratio", - options=StabilityAspectRatio, - default=StabilityAspectRatio.ratio_1_1, - tooltip="Aspect ratio of generated image.", - ), - IO.Combo.Input( - "style_preset", - options=get_stability_style_presets(), - tooltip="Optional desired style of generated image.", - advanced=True, - ), - IO.Float.Input( - "cfg_scale", - default=4.0, - min=1.0, - max=10.0, - step=0.1, - tooltip="How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)", - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.Image.Input( - "image", - optional=True, - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - IO.Float.Input( - "image_denoise", - default=0.5, - min=0.0, - max=1.0, - step=0.01, - tooltip="Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.", - optional=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model"]), - expr=""" - ( - $contains(widgets.model,"large") - ? {"type":"usd","usd":0.065} - : {"type":"usd","usd":0.035} - ) - """, - ), - ) - - @classmethod - async def execute( - cls, - model: str, - prompt: str, - aspect_ratio: str, - style_preset: str, - seed: int, - cfg_scale: float, - image: Optional[torch.Tensor] = None, - negative_prompt: str = "", - image_denoise: Optional[float] = 0.5, - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - # prepare image binary if image present - image_binary = None - mode = Stability_SD3_5_GenerationMode.text_to_image - if image is not None: - image_binary = tensor_to_bytesio(image, total_pixels=1504*1504).read() - mode = Stability_SD3_5_GenerationMode.image_to_image - aspect_ratio = None - else: - image_denoise = None - - if not negative_prompt: - negative_prompt = None - if style_preset == "None": - style_preset = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/generate/sd3", method="POST"), - response_model=StabilityStableUltraResponse, - data=StabilityStable3_5Request( - prompt=prompt, - negative_prompt=negative_prompt, - aspect_ratio=aspect_ratio, - seed=seed, - strength=image_denoise, - style_preset=style_preset, - cfg_scale=cfg_scale, - model=model, - mode=mode, - ), - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stable Diffusion 3.5 Image generation failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityUpscaleConservativeNode(IO.ComfyNode): - """ - Upscale image with minimal alterations to 4K resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityUpscaleConservativeNode", - display_name="Stability AI Upscale Conservative", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Image.Input("image"), - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", - ), - IO.Float.Input( - "creativity", - default=0.35, - min=0.2, - max=0.5, - step=0.01, - tooltip="Controls the likelihood of creating additional details not heavily conditioned by the init image.", - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.4}""", - ), - ) - - @classmethod - async def execute( - cls, - image: torch.Tensor, - prompt: str, - creativity: float, - seed: int, - negative_prompt: str = "", - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() - - if not negative_prompt: - negative_prompt = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/upscale/conservative", method="POST"), - response_model=StabilityStableUltraResponse, - data=StabilityUpscaleConservativeRequest( - prompt=prompt, - negative_prompt=negative_prompt, - creativity=round(creativity,2), - seed=seed, - ), - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stability Upscale Conservative generation failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityUpscaleCreativeNode(IO.ComfyNode): - """ - Upscale image with minimal alterations to 4K resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityUpscaleCreativeNode", - display_name="Stability AI Upscale Creative", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Image.Input("image"), - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", - ), - IO.Float.Input( - "creativity", - default=0.3, - min=0.1, - max=0.5, - step=0.01, - tooltip="Controls the likelihood of creating additional details not heavily conditioned by the init image.", - ), - IO.Combo.Input( - "style_preset", - options=get_stability_style_presets(), - tooltip="Optional desired style of generated image.", - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.6}""", - ), - ) - - @classmethod - async def execute( - cls, - image: torch.Tensor, - prompt: str, - creativity: float, - style_preset: str, - seed: int, - negative_prompt: str = "", - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() - - if not negative_prompt: - negative_prompt = None - if style_preset == "None": - style_preset = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/upscale/creative", method="POST"), - response_model=StabilityAsyncResponse, - data=StabilityUpscaleCreativeRequest( - prompt=prompt, - negative_prompt=negative_prompt, - creativity=round(creativity,2), - style_preset=style_preset, - seed=seed, - ), - files=files, - content_type="multipart/form-data", - ) - - response_poll = await poll_op( - cls, - ApiEndpoint(path=f"/proxy/stability/v2beta/results/{response_api.id}"), - response_model=StabilityResultsGetResponse, - poll_interval=3, - status_extractor=lambda x: get_async_dummy_status(x), - ) - - if response_poll.finish_reason != "SUCCESS": - raise Exception(f"Stability Upscale Creative generation failed: {response_poll.finish_reason}.") - - image_data = base64.b64decode(response_poll.result) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityUpscaleFastNode(IO.ComfyNode): - """ - Quickly upscales an image via Stability API call to 4x its original size; intended for upscaling low-quality/compressed images. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityUpscaleFastNode", - display_name="Stability AI Upscale Fast", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Image.Input("image"), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.02}""", - ), - ) - - @classmethod - async def execute(cls, image: torch.Tensor) -> IO.NodeOutput: - image_binary = tensor_to_bytesio(image, total_pixels=4096*4096).read() - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/upscale/fast", method="POST"), - response_model=StabilityStableUltraResponse, - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stability Upscale Fast failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityTextToAudio(IO.ComfyNode): - """Generates high-quality music and sound effects from text descriptions.""" - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityTextToAudio", - display_name="Stability AI Text To Audio", - category="partner/audio/Stability AI", - essentials_category="Audio", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Combo.Input( - "model", - options=["stable-audio-2.5"], - ), - IO.String.Input("prompt", multiline=True, default=""), - IO.Int.Input( - "duration", - default=190, - min=1, - max=190, - step=1, - tooltip="Controls the duration in seconds of the generated audio.", - optional=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for generation.", - optional=True, - ), - IO.Int.Input( - "steps", - default=8, - min=4, - max=8, - step=1, - tooltip="Controls the number of sampling steps.", - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Audio.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.2}""", - ), - ) - - @classmethod - async def execute(cls, model: str, prompt: str, duration: int, seed: int, steps: int) -> IO.NodeOutput: - validate_string(prompt, max_length=10000) - payload = StabilityTextToAudioRequest(prompt=prompt, model=model, duration=duration, seed=seed, steps=steps) - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/audio/stable-audio-2/text-to-audio", method="POST"), - response_model=StabilityAudioResponse, - data=payload, - content_type="multipart/form-data", - ) - if not response_api.audio: - raise ValueError("No audio file was received in response.") - return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response_api.audio))) - - -class StabilityAudioToAudio(IO.ComfyNode): - """Transforms existing audio samples into new high-quality compositions using text instructions.""" - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityAudioToAudio", - display_name="Stability AI Audio To Audio", - category="partner/audio/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Combo.Input( - "model", - options=["stable-audio-2.5"], - ), - IO.String.Input("prompt", multiline=True, default=""), - IO.Audio.Input("audio", tooltip="Audio must be between 6 and 190 seconds long."), - IO.Int.Input( - "duration", - default=190, - min=1, - max=190, - step=1, - tooltip="Controls the duration in seconds of the generated audio.", - optional=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for generation.", - optional=True, - ), - IO.Int.Input( - "steps", - default=8, - min=4, - max=8, - step=1, - tooltip="Controls the number of sampling steps.", - optional=True, - advanced=True, - ), - IO.Float.Input( - "strength", - default=1, - min=0.01, - max=1.0, - step=0.01, - display_mode=IO.NumberDisplay.slider, - tooltip="Parameter controls how much influence the audio parameter has on the generated audio.", - optional=True, - ), - ], - outputs=[ - IO.Audio.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.2}""", - ), - ) - - @classmethod - async def execute( - cls, model: str, prompt: str, audio: Input.Audio, duration: int, seed: int, steps: int, strength: float - ) -> IO.NodeOutput: - validate_string(prompt, max_length=10000) - validate_audio_duration(audio, 6, 190) - payload = StabilityAudioToAudioRequest( - prompt=prompt, model=model, duration=duration, seed=seed, steps=steps, strength=strength - ) - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/audio/stable-audio-2/audio-to-audio", method="POST"), - response_model=StabilityAudioResponse, - data=payload, - content_type="multipart/form-data", - files={"audio": audio_input_to_mp3(audio)}, - ) - if not response_api.audio: - raise ValueError("No audio file was received in response.") - return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response_api.audio))) - - -class StabilityAudioInpaint(IO.ComfyNode): - """Transforms part of existing audio sample using text instructions.""" - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityAudioInpaint", - display_name="Stability AI Audio Inpaint", - category="partner/audio/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Combo.Input( - "model", - options=["stable-audio-2.5"], - ), - IO.String.Input("prompt", multiline=True, default=""), - IO.Audio.Input("audio", tooltip="Audio must be between 6 and 190 seconds long."), - IO.Int.Input( - "duration", - default=190, - min=1, - max=190, - step=1, - tooltip="Controls the duration in seconds of the generated audio.", - optional=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for generation.", - optional=True, - ), - IO.Int.Input( - "steps", - default=8, - min=4, - max=8, - step=1, - tooltip="Controls the number of sampling steps.", - optional=True, - advanced=True, - ), - IO.Int.Input( - "mask_start", - default=30, - min=0, - max=190, - step=1, - optional=True, - advanced=True, - ), - IO.Int.Input( - "mask_end", - default=190, - min=0, - max=190, - step=1, - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Audio.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.2}""", - ), - ) - - @classmethod - async def execute( - cls, - model: str, - prompt: str, - audio: Input.Audio, - duration: int, - seed: int, - steps: int, - mask_start: int, - mask_end: int, - ) -> IO.NodeOutput: - validate_string(prompt, max_length=10000) - if mask_end <= mask_start: - raise ValueError(f"Value of mask_end({mask_end}) should be greater then mask_start({mask_start})") - validate_audio_duration(audio, 6, 190) - - payload = StabilityAudioInpaintRequest( - prompt=prompt, - model=model, - duration=duration, - seed=seed, - steps=steps, - mask_start=mask_start, - mask_end=mask_end, - ) - response_api = await sync_op( - cls, - endpoint=ApiEndpoint(path="/proxy/stability/v2beta/audio/stable-audio-2/inpaint", method="POST"), - response_model=StabilityAudioResponse, - data=payload, - content_type="multipart/form-data", - files={"audio": audio_input_to_mp3(audio)}, - ) - if not response_api.audio: - raise ValueError("No audio file was received in response.") - return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response_api.audio))) - - -class StabilityExtension(ComfyExtension): - @override - async def get_node_list(self) -> list[type[IO.ComfyNode]]: - return [ - StabilityStableImageUltraNode, - StabilityStableImageSD_3_5Node, - StabilityUpscaleConservativeNode, - StabilityUpscaleCreativeNode, - StabilityUpscaleFastNode, - StabilityTextToAudio, - StabilityAudioToAudio, - StabilityAudioInpaint, - ] - - -async def comfy_entrypoint() -> StabilityExtension: - return StabilityExtension() diff --git a/comfy_api_nodes/nodes_sync_so.py b/comfy_api_nodes/nodes_sync_so.py new file mode 100644 index 000000000..27382b399 --- /dev/null +++ b/comfy_api_nodes/nodes_sync_so.py @@ -0,0 +1,391 @@ +from typing_extensions import override + +from comfy_api.latest import IO, ComfyExtension, Input +from comfy_api_nodes.apis.sync_so import ( + SyncActiveSpeakerDetection, + SyncGeneration, + SyncGenerationOptions, + SyncGenerationRequest, + SyncInputItem, +) +from comfy_api_nodes.util import ( + ApiEndpoint, + download_url_to_video_output, + downscale_image_tensor, + downscale_image_tensor_by_max_side, + get_image_dimensions, + get_number_of_images, + poll_op, + sync_op, + upload_audio_to_comfyapi, + upload_image_to_comfyapi, + upload_video_to_comfyapi, + validate_audio_duration, +) + + +class SyncLipSyncNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="SyncLipSyncNode", + display_name="sync.so Lip Sync", + category="partner/video/sync.so", + description=( + "Re-sync mouth movement in a video to new speech audio using sync.so. " + "Handles close-ups, profiles and obstructions automatically while preserving " + "the speaker's expression. Cost scales with output duration." + ), + inputs=[ + IO.Video.Input( + "video", + tooltip="Footage of the speaker to re-sync. Up to 4K (4096x2160); " + "a constant frame rate of 24/25/30 fps works best.", + ), + IO.Audio.Input( + "audio", + tooltip="Speech audio to sync the mouth to.", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option( + "sync-3", + [ + IO.Combo.Input( + "sync_mode", + options=["bounce", "cut_off", "loop", "silence", "remap"], + default="bounce", + tooltip=( + "How to handle a duration mismatch between video and audio; " + "this also sets the output length. " + "bounce: video plays forward then backward until the audio ends " + "(output = audio length). " + "loop: video restarts until the audio ends (output = audio length). " + "remap: video is time-stretched to match the audio (output = audio length). " + "cut_off: the longer track is trimmed (output = shorter length). " + "silence: nothing is trimmed; the shorter track is padded " + "(output = longer length)." + ), + ), + IO.Combo.Input( + "speaker_selection", + options=["default", "auto-detect", "coordinates"], + default="default", + tooltip=( + "Which face to lipsync when several people are visible. " + "default: let the model decide. " + "auto-detect: detect and follow the active speaker. " + "coordinates: target the face at pixel (speaker_x, speaker_y) " + "in the frame chosen by speaker_frame." + ), + ), + IO.Int.Input( + "speaker_frame", + default=0, + min=0, + max=1_000_000, + advanced=True, + tooltip="Video frame used to locate the speaker. " + "Only used when speaker_selection is 'coordinates'.", + ), + IO.Int.Input( + "speaker_x", + default=0, + min=0, + max=4096, + advanced=True, + tooltip="X pixel coordinate of the speaker's face. " + "Only used when speaker_selection is 'coordinates'.", + ), + IO.Int.Input( + "speaker_y", + default=0, + min=0, + max=4096, + advanced=True, + tooltip="Y pixel coordinate of the speaker's face. " + "Only used when speaker_selection is 'coordinates'.", + ), + ], + ) + ], + tooltip="sync.so generation model.", + ), + ], + outputs=[IO.Video.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd":0.19019,"format":{"approximate":true,"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + video: Input.Video, + audio: Input.Audio, + seed: int, + model: dict, + ) -> IO.NodeOutput: + try: + width, height = video.get_dimensions() + except Exception: + width = height = None + if width and height and (max(width, height) > 4096 or width * height > 4096 * 2160): + raise ValueError( + f"sync.so rejects videos above 4K (4096x2160); got {width}x{height}. Downscale the video first." + ) + validate_audio_duration(audio, max_duration=600) + + if model["speaker_selection"] == "auto-detect": + speaker_detection = SyncActiveSpeakerDetection(auto_detect=True) + elif model["speaker_selection"] == "coordinates": + speaker_detection = SyncActiveSpeakerDetection( + frame_number=model["speaker_frame"], + coordinates=[model["speaker_x"], model["speaker_y"]], + ) + else: + speaker_detection = None + + video_url = await upload_video_to_comfyapi(cls, video, max_duration=600) + audio_url = await upload_audio_to_comfyapi(cls, audio) + + generation = await sync_op( + cls, + ApiEndpoint(path="/proxy/synclabs/v2/generate", method="POST"), + response_model=SyncGeneration, + data=SyncGenerationRequest( + model=model["model"], + input=[ + SyncInputItem(type="video", url=video_url), + SyncInputItem(type="audio", url=audio_url), + ], + options=SyncGenerationOptions( + sync_mode=model["sync_mode"], + active_speaker_detection=speaker_detection, + ), + ), + ) + generation = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/synclabs/v2/generate/{generation.id}"), + response_model=SyncGeneration, + status_extractor=lambda g: g.status, + completed_statuses=["COMPLETED", "FAILED", "REJECTED"], + failed_statuses=[], + queued_statuses=["PENDING"], + poll_interval=10.0, + ) + if generation.status != "COMPLETED": + code = f" [{generation.errorCode}]" if generation.errorCode else "" + raise ValueError( + f"sync.so generation {generation.status.lower()}{code}: " + f"{generation.error or 'no error details provided'}" + ) + if not generation.outputUrl: + raise ValueError("sync.so generation completed but no output URL was returned.") + return IO.NodeOutput(await download_url_to_video_output(generation.outputUrl)) + + +class SyncTalkingImageNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="SyncTalkingImageNode", + display_name="sync.so Talking Image", + category="partner/video/sync.so", + description=( + "Animate a still portrait into a talking video driven by speech audio, " + "using sync.so's sync-3 model. The output duration matches the audio. " + "Cost scales with output duration." + ), + inputs=[ + IO.Image.Input( + "image", + tooltip="A single image with a clearly visible face, up to 4K (4096x2160).", + ), + IO.Audio.Input( + "audio", + tooltip="Speech audio driving the talking video; the output duration matches it. " + "Chain any TTS node here to drive the animation from text.", + ), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Optional guidance for how the portrait comes to life, e.g. " + "'make the subject smile and look at the camera'. " + "Leave empty for natural talking motion.", + ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option( + "sync-3", + [ + IO.Combo.Input( + "speaker_selection", + options=["default", "coordinates"], + default="default", + tooltip=( + "Which face to animate when several people are visible. " + "default: let the model decide. " + "coordinates: target the face at pixel (speaker_x, speaker_y) " + "in the image. Auto-detection is not supported for images." + ), + ), + IO.Int.Input( + "speaker_x", + default=0, + min=0, + max=4096, + advanced=True, + tooltip="X pixel coordinate of the speaker's face. " + "Only used when speaker_selection is 'coordinates'.", + ), + IO.Int.Input( + "speaker_y", + default=0, + min=0, + max=4096, + advanced=True, + tooltip="Y pixel coordinate of the speaker's face. " + "Only used when speaker_selection is 'coordinates'.", + ), + IO.Boolean.Input( + "auto_downscale", + default=True, + advanced=True, + tooltip="Automatically downscale the image if it exceeds the 4K " + "(4096x2160) input limit; speaker coordinates are scaled to match. " + "When disabled, an oversized image raises an error instead.", + ), + ], + ) + ], + tooltip="sync.so generation model. Image input is exclusive to sync-3.", + ), + ], + outputs=[IO.Video.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd":0.19019,"format":{"approximate":true,"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + image: Input.Image, + audio: Input.Audio, + prompt: str, + seed: int, + model: dict, + ) -> IO.NodeOutput: + if get_number_of_images(image) != 1: + raise ValueError("Exactly one image is required; got a batch. Pick one frame first.") + validate_audio_duration(audio, max_duration=600) + + height, width = get_image_dimensions(image) + speaker_x, speaker_y = model["speaker_x"], model["speaker_y"] + if max(width, height) > 4096 or width * height > 4096 * 2160: + if not model["auto_downscale"]: + raise ValueError( + f"sync.so rejects images above 4K (4096x2160); got {width}x{height}. " + "Downscale the image first or enable auto_downscale." + ) + image = downscale_image_tensor(image, total_pixels=4096 * 2160) + image = downscale_image_tensor_by_max_side(image, max_side=4096) + new_height, new_width = get_image_dimensions(image) + # speaker coordinates are given in the original image's pixel space + speaker_x = min(new_width - 1, round(speaker_x * new_width / width)) + speaker_y = min(new_height - 1, round(speaker_y * new_height / height)) + + if model["speaker_selection"] == "coordinates": + speaker_detection = SyncActiveSpeakerDetection( + frame_number=0, # images have a single frame; auto_detect is rejected by the API + coordinates=[speaker_x, speaker_y], + ) + else: + speaker_detection = None + + image_url = await upload_image_to_comfyapi(cls, image, mime_type="image/png", total_pixels=None) + audio_url = await upload_audio_to_comfyapi(cls, audio) + + generation = await sync_op( + cls, + ApiEndpoint(path="/proxy/synclabs/v2/generate", method="POST"), + response_model=SyncGeneration, + data=SyncGenerationRequest( + model=model["model"], + input=[ + SyncInputItem(type="image", url=image_url), + SyncInputItem(type="audio", url=audio_url), + ], + options=SyncGenerationOptions( + i2v_prompt=prompt.strip() or None, + active_speaker_detection=speaker_detection, + ), + ), + ) + generation = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/synclabs/v2/generate/{generation.id}"), + response_model=SyncGeneration, + status_extractor=lambda g: g.status, + completed_statuses=["COMPLETED", "FAILED", "REJECTED"], + failed_statuses=[], + queued_statuses=["PENDING"], + poll_interval=10.0, + ) + if generation.status != "COMPLETED": + code = f" [{generation.errorCode}]" if generation.errorCode else "" + raise ValueError( + f"sync.so generation {generation.status.lower()}{code}: " + f"{generation.error or 'no error details provided'}" + ) + if not generation.outputUrl: + raise ValueError("sync.so generation completed but no output URL was returned.") + return IO.NodeOutput(await download_url_to_video_output(generation.outputUrl)) + + +class SyncExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + SyncLipSyncNode, + SyncTalkingImageNode, + ] + + +async def comfy_entrypoint() -> SyncExtension: + return SyncExtension() diff --git a/comfy_api_nodes/nodes_tripo.py b/comfy_api_nodes/nodes_tripo.py index a3f2cb053..228fe8a1d 100644 --- a/comfy_api_nodes/nodes_tripo.py +++ b/comfy_api_nodes/nodes_tripo.py @@ -1,6 +1,6 @@ from typing_extensions import override -from comfy_api.latest import IO, ComfyExtension, Input +from comfy_api.latest import IO, ComfyExtension, Input, Types from comfy_api_nodes.apis.tripo import ( TripoAnimateRetargetRequest, TripoAnimateRigRequest, @@ -8,6 +8,7 @@ from comfy_api_nodes.apis.tripo import ( TripoFileEmptyReference, TripoFileReference, TripoImageToModelRequest, + TripoImportModelRequest, TripoModelVersion, TripoMultiviewToModelRequest, TripoOrientation, @@ -21,6 +22,7 @@ from comfy_api_nodes.apis.tripo import ( TripoTaskType, TripoTextToModelRequest, TripoTextureModelRequest, + TripoTexturePrompt, TripoUrlReference, ) from comfy_api_nodes.util import ( @@ -28,6 +30,7 @@ from comfy_api_nodes.util import ( download_url_to_file_3d, poll_op, sync_op, + upload_3d_model_to_comfyapi, upload_images_to_comfyapi, ) @@ -538,6 +541,14 @@ class TripoTextureNode(IO.ComfyNode): optional=True, advanced=True, ), + IO.String.Input( + "texture_prompt", + default="", + multiline=True, + optional=True, + tooltip="Optional text guidance for texturing. Required in practice for imported " + "models (Tripo: Import Model), which carry no source image to infer colors from.", + ), ], outputs=[ IO.String.Output(display_name="model_file"), # for backward compatibility only @@ -571,6 +582,7 @@ class TripoTextureNode(IO.ComfyNode): texture_seed: int | None = None, texture_quality: str | None = None, texture_alignment: str | None = None, + texture_prompt: str = "", ) -> IO.NodeOutput: response = await sync_op( cls, @@ -583,6 +595,7 @@ class TripoTextureNode(IO.ComfyNode): texture_seed=texture_seed, texture_quality=texture_quality, texture_alignment=texture_alignment, + texture_prompt=TripoTexturePrompt(text=texture_prompt.strip()) if texture_prompt.strip() else None, ), ) return await poll_until_finished(cls, response, average_duration=80) @@ -915,6 +928,90 @@ class TripoConversionNode(IO.ComfyNode): return await poll_until_finished(cls, response, average_duration=30) +class TripoImportModelNode(IO.ComfyNode): + """Imports an external 3D model into Tripo, producing a MODEL_TASK_ID for post-processing nodes.""" + + SUPPORTED_FORMATS = ("glb", "fbx", "obj", "stl") + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TripoImportModelNode", + display_name="Tripo: Import Model", + category="partner/3d/Tripo", + description="Import an external 3D model (e.g. from Rodin, Hunyuan3D or a local file) into Tripo " + "to use it with Tripo's post-processing nodes: Texture, Rig, Convert. " + "GLB is recommended: textures survive import only when embedded in the file. " + "Note that texturing an imported model requires a texture prompt.", + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[IO.File3DGLB, IO.File3DFBX, IO.File3DOBJ, IO.File3DSTL, IO.File3DAny], + tooltip="3D model to import (GLB / FBX / OBJ / STL, up to 150 MB). " + "OBJ and STL files carry no embedded textures.", + ), + ], + outputs=[ + IO.Custom("MODEL_TASK_ID").Output(display_name="model task_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"text","text":"Free"}""", + ), + ) + + @classmethod + async def execute(cls, model_3d: Types.File3D) -> IO.NodeOutput: + file_format = (model_3d.format or "").lstrip(".").lower() + if file_format == "gltf": + raise ValueError( + "GLTF (.gltf) references external files and cannot be imported. Export a single-file GLB instead." + ) + if file_format not in cls.SUPPORTED_FORMATS: + raise ValueError( + f"Unsupported 3D format '{file_format or 'unknown'}'. " + f"Tripo import supports: {', '.join(f.upper() for f in cls.SUPPORTED_FORMATS)}." + ) + size = len(model_3d.get_bytes()) + if size > 150 * 1024 * 1024: + raise ValueError(f"Model file is {size / (1024 * 1024):.1f} MB; Tripo import allows up to 150 MB.") + + url = await upload_3d_model_to_comfyapi(cls, model_3d, file_format) + response = await sync_op( + cls, + endpoint=ApiEndpoint(path="/proxy/tripo/v2/openapi/import", method="POST"), + response_model=TripoTaskResponse, + data=TripoImportModelRequest(url=url, format=file_format), + ) + if response.code != 0: + raise RuntimeError(f"Failed to import model: {response.error}") + + task_id = response.data.task_id + response_poll = await poll_op( + cls, + poll_endpoint=ApiEndpoint(path=f"/proxy/tripo/v2/openapi/task/{task_id}"), + response_model=TripoTaskResponse, + failed_statuses=[ + TripoTaskStatus.FAILED, + TripoTaskStatus.CANCELLED, + TripoTaskStatus.UNKNOWN, + TripoTaskStatus.BANNED, + TripoTaskStatus.EXPIRED, + ], + status_extractor=lambda x: x.data.status, + progress_extractor=lambda x: x.data.progress, + estimated_duration=10, + ) + if response_poll.data.status != TripoTaskStatus.SUCCESS: + raise RuntimeError(f"Failed to import model: {response_poll}") + return IO.NodeOutput(task_id) + + def _p1_price_expr(*, geometry_credits: int, textured_credits: int, detailed_credits: int) -> str: return ( "(" @@ -1292,6 +1389,7 @@ class TripoExtension(ComfyExtension): TripoP1TextToModelNode, TripoP1ImageToModelNode, TripoP1MultiviewToModelNode, + TripoImportModelNode, TripoTextureNode, TripoRefineNode, TripoRigNode, diff --git a/comfy_api_nodes/nodes_wan.py b/comfy_api_nodes/nodes_wan.py index b7b97d70f..1782739fd 100644 --- a/comfy_api_nodes/nodes_wan.py +++ b/comfy_api_nodes/nodes_wan.py @@ -48,10 +48,13 @@ from comfy_api_nodes.util import ( upload_image_to_comfyapi, upload_video_to_comfyapi, validate_audio_duration, + validate_image_aspect_ratio, + validate_image_dimensions, validate_string, validate_video_duration, ) + RES_IN_PARENS = re.compile(r"\((\d+)\s*[x×]\s*(\d+)\)") @@ -1657,6 +1660,44 @@ class HappyHorseTextToVideoApi(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "happyhorse-1.1-t2v", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Prompt describing the elements and visual features. " + "Supports English and Chinese.", + ), + IO.Combo.Input( + "resolution", + options=["720P", "1080P"], + ), + IO.Combo.Input( + "ratio", + options=[ + "16:9", + "9:16", + "1:1", + "4:3", + "3:4", + "21:9", + "9:21", + "5:4", + "4:5", + ], + ), + IO.Int.Input( + "duration", + default=5, + min=3, + max=15, + step=1, + display_mode=IO.NumberDisplay.number, + ), + ], + ), IO.DynamicCombo.Option( "happyhorse-1.0-t2v", [ @@ -1719,7 +1760,9 @@ class HappyHorseTextToVideoApi(IO.ComfyNode): ( $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $ppsTable := { "720p": 0.14, "1080p": 0.24 }; + $ppsTable := $contains(widgets.model, "1.1") + ? { "720p": 0.2002, "1080p": 0.2574 } + : { "720p": 0.14, "1080p": 0.24 }; $pps := $lookup($ppsTable, $res); { "type": "usd", "usd": $pps * $dur } ) @@ -1781,6 +1824,30 @@ class HappyHorseImageToVideoApi(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "happyhorse-1.1-i2v", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Prompt describing the elements and visual features. " + "Supports English and Chinese.", + ), + IO.Combo.Input( + "resolution", + options=["720P", "1080P"], + ), + IO.Int.Input( + "duration", + default=5, + min=3, + max=15, + step=1, + display_mode=IO.NumberDisplay.number, + ), + ], + ), IO.DynamicCombo.Option( "happyhorse-1.0-i2v", [ @@ -1843,7 +1910,9 @@ class HappyHorseImageToVideoApi(IO.ComfyNode): ( $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $ppsTable := { "720p": 0.14, "1080p": 0.24 }; + $ppsTable := $contains(widgets.model, "1.1") + ? { "720p": 0.2002, "1080p": 0.2574 } + : { "720p": 0.14, "1080p": 0.24 }; $pps := $lookup($ppsTable, $res); { "type": "usd", "usd": $pps * $dur } ) @@ -1859,6 +1928,8 @@ class HappyHorseImageToVideoApi(IO.ComfyNode): seed: int, watermark: bool, ): + validate_image_dimensions(first_frame, min_width=300, min_height=300) + validate_image_aspect_ratio(first_frame, (1, 2.5), (2.5, 1), strict=False) media = [ Wan27MediaItem( type="first_frame", @@ -2053,6 +2124,62 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "happyhorse-1.1-r2v", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Prompt describing the video. Use identifiers such as 'character1' and " + "'character2' to refer to the reference characters.", + ), + IO.Combo.Input( + "resolution", + options=["720P", "1080P"], + ), + IO.Combo.Input( + "ratio", + options=[ + "16:9", + "9:16", + "1:1", + "4:3", + "3:4", + "21:9", + "9:21", + "5:4", + "4:5", + ], + ), + IO.Int.Input( + "duration", + default=5, + min=3, + max=15, + step=1, + display_mode=IO.NumberDisplay.number, + ), + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("reference_image"), + names=[ + "image1", + "image2", + "image3", + "image4", + "image5", + "image6", + "image7", + "image8", + "image9", + ], + min=1, + ), + ), + ], + ), IO.DynamicCombo.Option( "happyhorse-1.0-r2v", [ @@ -2133,7 +2260,9 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): ( $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $ppsTable := { "720p": 0.14, "1080p": 0.24 }; + $ppsTable := $contains(widgets.model, "1.1") + ? { "720p": 0.2002, "1080p": 0.2574 } + : { "720p": 0.14, "1080p": 0.24 }; $pps := $lookup($ppsTable, $res); { "type": "usd", "usd": $pps * $dur } ) @@ -2149,8 +2278,11 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): watermark: bool, ): validate_string(model["prompt"], strip_whitespace=False, min_length=1) - media = [] reference_images = model.get("reference_images", {}) + for key in reference_images: + validate_image_dimensions(reference_images[key], min_width=400, min_height=400) + validate_image_aspect_ratio(reference_images[key], (1, 2.5), (2.5, 1), strict=False) + media = [] for key in reference_images: media.append( Wan27MediaItem( @@ -2159,7 +2291,7 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): ) ) if not media: - raise ValueError("At least one reference reference image must be provided.") + raise ValueError("At least one reference image must be provided.") initial_response = await sync_op( cls, diff --git a/comfy_api_nodes/util/__init__.py b/comfy_api_nodes/util/__init__.py index 25cb88869..1fb6b96cf 100644 --- a/comfy_api_nodes/util/__init__.py +++ b/comfy_api_nodes/util/__init__.py @@ -26,6 +26,7 @@ from .conversions import ( text_filepath_to_base64_string, text_filepath_to_data_uri, trim_video, + upscale_image_tensor_to_min_pixels, upscale_video_to_min_pixels, video_to_base64_string, ) @@ -99,6 +100,7 @@ __all__ = [ "text_filepath_to_base64_string", "text_filepath_to_data_uri", "trim_video", + "upscale_image_tensor_to_min_pixels", "upscale_video_to_min_pixels", "video_to_base64_string", # Validation utilities diff --git a/comfy_api_nodes/util/_helpers.py b/comfy_api_nodes/util/_helpers.py index 648defe3d..acab10d95 100644 --- a/comfy_api_nodes/util/_helpers.py +++ b/comfy_api_nodes/util/_helpers.py @@ -4,13 +4,18 @@ import os import re import time from collections.abc import Callable +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime from io import BytesIO from yarl import URL from comfy.cli_args import args +from comfy.comfy_api_env import normalize_comfy_api_base +from comfy.deploy_environment import get_deploy_environment from comfy.model_management import processing_interrupted from comfy_api.latest import IO +from comfyui_version import __version__ as comfyui_version from .common_exceptions import ProcessingInterrupted @@ -35,8 +40,33 @@ def get_auth_header(node_cls: type[IO.ComfyNode]) -> dict[str, str]: return {} +def get_usage_source(node_cls: type[IO.ComfyNode]) -> str: + """Source of the prompt that triggered this API node. + + Defaults to "comfyui-api" when the submitting client didn't identify itself, + i.e. a direct API call to this server. + """ + return node_cls.hidden.comfy_usage_source or "comfyui-api" + + +def get_comfy_api_headers(node_cls: type[IO.ComfyNode]) -> dict[str, str]: + """Common headers (auth, deploy environment, usage source) for Comfy API requests. + + Centralizes the shared header set so every Comfy API request sends a consistent + set and new shared headers only need to be added in one place. Intended for + relative/cloud URLs resolved against ``default_base_url()``; because the result + includes auth, callers must not attach it to arbitrary absolute/presigned URLs. + """ + return { + **get_auth_header(node_cls), + "Comfy-Env": get_deploy_environment(), + "Comfy-Usage-Source": get_usage_source(node_cls), + "Comfy-Core-Version": comfyui_version, + } + + def default_base_url() -> str: - return getattr(args, "comfy_api_base", "https://api.comfy.org") + return normalize_comfy_api_base(getattr(args, "comfy_api_base", "https://api.comfy.org")) async def sleep_with_interrupt( @@ -66,6 +96,32 @@ async def sleep_with_interrupt( await asyncio.sleep(min(1.0, end - now)) +def _retry_after_wait(value: str | None, fallback: float, max_wait: float) -> float: + """Delay before the next retry, honoring a server ``Retry-After`` header.""" + + seconds: float | None = None + if value is not None: + value = value.strip() + if value.isascii() and value.isdigit(): + # delay-seconds form. The ASCII-digit guard keeps exotic Unicode "digit" characters away from float() + # an all-digit string always converts (huge values become inf, never raising). + seconds = float(value) + elif value: + # HTTP-date form. parsedate_to_datetime raises OverflowError (not a ValueError) on absurd years/offsets + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + parsed = None + if parsed is not None: + if parsed.tzinfo is None: # naive datetime: HTTP-date is UTC + parsed = parsed.replace(tzinfo=timezone.utc) + delta = (parsed - datetime.now(timezone.utc)).total_seconds() + seconds = delta if delta > 0 else 0.0 + if seconds is None: + return fallback + return min(seconds, max_wait) + + def mimetype_to_extension(mime_type: str) -> str: """Converts a MIME type to a file extension.""" return mime_type.split("/")[-1].lower() diff --git a/comfy_api_nodes/util/client.py b/comfy_api_nodes/util/client.py index 57c501724..66aab17f8 100644 --- a/comfy_api_nodes/util/client.py +++ b/comfy_api_nodes/util/client.py @@ -19,12 +19,11 @@ from comfy import utils from comfy_api.latest import IO from server import PromptServer -from comfy.deploy_environment import get_deploy_environment - from . import request_logger from ._helpers import ( + _retry_after_wait, default_base_url, - get_auth_header, + get_comfy_api_headers, get_node_id, is_processing_interrupted, sleep_with_interrupt, @@ -84,6 +83,7 @@ class _PollUIState: _RETRY_STATUS = {408, 500, 502, 503, 504} # status 429 is handled separately +_MAX_RETRY_AFTER_WAIT = 150.0 # Cap a server Retry-After at this many seconds so a large hint can't block execution COMPLETED_STATUSES = ["succeeded", "succeed", "success", "completed", "finished", "done", "complete"] FAILED_STATUSES = ["cancelled", "canceled", "canceling", "fail", "failed", "error"] QUEUED_STATUSES = ["created", "queued", "queueing", "submitted", "initializing", "wait", "in_queue"] @@ -645,8 +645,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool): payload_headers = {"Accept": "*/*"} if expect_binary else {"Accept": "application/json"} if not parsed_url.scheme and not parsed_url.netloc: # is URL relative? - payload_headers.update(get_auth_header(cfg.node_cls)) - payload_headers["Comfy-Env"] = get_deploy_environment() + payload_headers.update(get_comfy_api_headers(cfg.node_cls)) if cfg.endpoint.headers: payload_headers.update(cfg.endpoint.headers) @@ -750,6 +749,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool): should_retry = True if should_retry: + wait_time = _retry_after_wait(resp.headers.get("Retry-After"), wait_time, _MAX_RETRY_AFTER_WAIT) logging.warning( "HTTP %s %s -> %s. Waiting %.2fs (%s).", method, diff --git a/comfy_api_nodes/util/conversions.py b/comfy_api_nodes/util/conversions.py index a1b5d599c..9cd644fc0 100644 --- a/comfy_api_nodes/util/conversions.py +++ b/comfy_api_nodes/util/conversions.py @@ -448,6 +448,15 @@ def _compute_upscale_dims(src_w: int, src_h: int, total_pixels: int) -> tuple[in return new_w, new_h +def upscale_image_tensor_to_min_pixels(image: torch.Tensor, total_pixels: int) -> torch.Tensor: + samples = image.movedim(-1, 1) + dims = _compute_upscale_dims(samples.shape[3], samples.shape[2], int(total_pixels)) + if dims is None: + return image + new_w, new_h = dims + return common_upscale(samples, new_w, new_h, "lanczos", "disabled").movedim(1, -1) + + def upscale_video_to_min_pixels(video: Input.Video, min_pixels: int) -> Input.Video: """Upscale a video to meet at least ``min_pixels`` (w * h), preserving aspect ratio. diff --git a/comfy_api_nodes/util/download_helpers.py b/comfy_api_nodes/util/download_helpers.py index aa588d038..0ec3c6e66 100644 --- a/comfy_api_nodes/util/download_helpers.py +++ b/comfy_api_nodes/util/download_helpers.py @@ -17,7 +17,7 @@ from folder_paths import get_output_directory from . import request_logger from ._helpers import ( default_base_url, - get_auth_header, + get_comfy_api_headers, is_processing_interrupted, sleep_with_interrupt, to_aiohttp_url, @@ -64,7 +64,7 @@ async def download_url_to_bytesio( if cls is None: raise ValueError("For relative 'cloud' paths, the `cls` parameter is required.") url = urljoin(default_base_url().rstrip("/") + "/", url.lstrip("/")) - headers = get_auth_header(cls) + headers = get_comfy_api_headers(cls) while True: attempt += 1 diff --git a/comfy_api_nodes/util/request_logger.py b/comfy_api_nodes/util/request_logger.py index fe0543d9b..70ecaf41a 100644 --- a/comfy_api_nodes/util/request_logger.py +++ b/comfy_api_nodes/util/request_logger.py @@ -9,6 +9,7 @@ from typing import Any import folder_paths logger = logging.getLogger(__name__) +_SENSITIVE_HEADERS = {"authorization", "x-api-key"} def get_log_directory(): @@ -73,6 +74,10 @@ def _format_data_for_logging(data: Any) -> str: return str(data) +def _redact_headers(headers: dict) -> dict: + return {k: ("***" if k.lower() in _SENSITIVE_HEADERS else v) for k, v in headers.items()} + + def log_request_response( operation_id: str, request_method: str, @@ -101,7 +106,7 @@ def log_request_response( log_content.append(f"Method: {request_method}") log_content.append(f"URL: {request_url}") if request_headers: - log_content.append(f"Headers:\n{_format_data_for_logging(request_headers)}") + log_content.append(f"Headers:\n{_format_data_for_logging(_redact_headers(request_headers))}") if request_params: log_content.append(f"Params:\n{_format_data_for_logging(request_params)}") if request_data is not None: diff --git a/comfy_api_nodes/util/upload_helpers.py b/comfy_api_nodes/util/upload_helpers.py index 6d1d107a1..f7029ee78 100644 --- a/comfy_api_nodes/util/upload_helpers.py +++ b/comfy_api_nodes/util/upload_helpers.py @@ -158,7 +158,14 @@ async def upload_video_to_comfyapi( # Convert VideoInput to BytesIO using specified container/codec video_bytes_io = BytesIO() - video.save_to(video_bytes_io, format=container, codec=codec) + try: + video.save_to(video_bytes_io, format=container, codec=codec) + except Exception as e: + raise ValueError( + f"Could not convert the input video to {container.value.upper()} for upload; " + f"the file may be corrupted or use an unsupported codec. " + f"Try re-exporting it as MP4 (H.264). Original error: {e}" + ) from e video_bytes_io.seek(0) return await upload_file_to_comfyapi(cls, video_bytes_io, filename, upload_mime_type, wait_label) diff --git a/comfy_execution/asset_enrichment.py b/comfy_execution/asset_enrichment.py new file mode 100644 index 000000000..38e9496a8 --- /dev/null +++ b/comfy_execution/asset_enrichment.py @@ -0,0 +1,66 @@ +"""Enrich executed-node output entries with asset id.""" +import logging +import os + + +def enrich_output_with_assets(output_ui: dict) -> dict: + """Register file-type output entries as assets and inject their ``id``. + + Runs at output-processing time, once per produced output, when + --enable-assets is set. Returns a new dict; entries without a resolvable + on-disk file path are left unchanged. Errors are caught per-entry so a + failure never blocks execution or the other entries. + """ + from comfy.cli_args import args + if not args.enable_assets: + return output_ui + + import folder_paths + from app.assets.services.ingest import register_file_in_place, DependencyMissingError + + enriched = {} + for key, entries in output_ui.items(): + if not isinstance(entries, list): + enriched[key] = entries + continue + new_entries = [] + for entry in entries: + if not isinstance(entry, dict) or "filename" not in entry or "type" not in entry: + new_entries.append(entry) + continue + try: + base = folder_paths.get_directory_by_type(entry["type"]) + if base is None: + new_entries.append(entry) + continue + base_abs = os.path.abspath(base) + abs_path = os.path.abspath(os.path.join(base_abs, entry.get("subfolder") or "", entry["filename"])) + try: + if os.path.commonpath([base_abs, abs_path]) != base_abs: + raise ValueError("escapes base") + except ValueError: + logging.warning("Asset enrichment skipped (path escapes base): %s", entry.get("filename")) + new_entries.append(entry) + continue + if not os.path.isfile(abs_path): + new_entries.append(entry) + continue + + # Register unconditionally: the file was just produced, and + # register_file_in_place re-hashes so an overwritten path can + # never carry a stale id. + result = register_file_in_place( + abs_path=abs_path, + name=entry["filename"], + tags=[entry["type"]], + ) + + entry = dict(entry) + entry["id"] = result.ref.id + except DependencyMissingError: + logging.warning("Asset enrichment skipped (blake3 not available): %s", entry.get("filename")) + except Exception: + logging.warning("Failed to enrich output entry with asset id: %s", entry.get("filename"), exc_info=True) + new_entries.append(entry) + enriched[key] = new_entries + return enriched diff --git a/comfy_execution/caching.py b/comfy_execution/caching.py index ba1e8bc84..6bd99b68f 100644 --- a/comfy_execution/caching.py +++ b/comfy_execution/caching.py @@ -503,6 +503,22 @@ RAM_CACHE_DEFAULT_RAM_USAGE = 0.05 RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER = 1.3 +RAM_CACHE_LARGE_INTERMEDIATE = 512 * 1024 ** 2 + + +def all_outputs_dynamic(outputs): + if outputs is None: + return False + + for output in outputs: + if isinstance(output, (list, tuple)): + if not all_outputs_dynamic(output): + return False + elif not hasattr(output, "is_dynamic") or not output.is_dynamic(): + return False + + return True + class RAMPressureCache(LRUCache): def __init__(self, key_class, enable_providers=False): @@ -524,20 +540,25 @@ class RAMPressureCache(LRUCache): self.timestamps[self.cache_key_set.get_data_key(node_id)] = time.time() super().set_local(node_id, value) - def ram_release(self, target, free_active=False): + def ram_release(self, target, free_active=False, min_entry_size=0): if psutil.virtual_memory().available >= target: - return + return 0 clean_list = [] for key, cache_entry in self.cache.items(): if not free_active and self.used_generation[key] == self.generation: continue - oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key]) + + if all_outputs_dynamic(cache_entry.outputs) and self.used_generation[key] == self.generation: + continue + + oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key]) ram_usage = RAM_CACHE_DEFAULT_RAM_USAGE + oom_ram_usage = ram_usage def scan_list_for_ram_usage(outputs): - nonlocal ram_usage + nonlocal ram_usage, oom_ram_usage if outputs is None: return for output in outputs: @@ -545,19 +566,26 @@ class RAMPressureCache(LRUCache): scan_list_for_ram_usage(output) elif isinstance(output, torch.Tensor) and output.device.type == 'cpu': ram_usage += output.numel() * output.element_size() + oom_ram_usage += output.numel() * output.element_size() elif isinstance(output, ModelPatcher) and self.used_generation[key] != self.generation: #old ModelPatchers are the first to go - ram_usage = 1e30 + oom_ram_usage = 1e30 scan_list_for_ram_usage(cache_entry.outputs) - oom_score *= ram_usage + if ram_usage < min_entry_size: + continue + + oom_score *= oom_ram_usage #In the case where we have no information on the node ram usage at all, #break OOM score ties on the last touch timestamp (pure LRU) - bisect.insort(clean_list, (oom_score, self.timestamps[key], key)) + bisect.insort(clean_list, (oom_score, self.timestamps[key], key, ram_usage)) + freed = 0 while psutil.virtual_memory().available < target and clean_list: - _, _, key = clean_list.pop() + _, _, key, ram_usage = clean_list.pop() del self.cache[key] self.used_generation.pop(key, None) self.timestamps.pop(key, None) self.children.pop(key, None) + freed += ram_usage + return freed diff --git a/comfy_execution/jobs.py b/comfy_execution/jobs.py index fcd7ef735..f0ad59f86 100644 --- a/comfy_execution/jobs.py +++ b/comfy_execution/jobs.py @@ -3,11 +3,23 @@ Job utilities for the /api/jobs endpoint. Provides normalization and helper functions for job status tracking. """ -from typing import Optional +import uuid +from typing import Callable, Optional from comfy_api.internal import prune_dict +# Result of classifying a job for cancellation. +# 'running' -> job is currently executing (interrupt it) +# 'pending' -> job is queued but not started (dequeue it) +# 'terminal' -> job already finished (present in history); cancel is a no-op +# 'unknown' -> job id is not present anywhere +CANCEL_RUNNING = 'running' +CANCEL_PENDING = 'pending' +CANCEL_TERMINAL = 'terminal' +CANCEL_UNKNOWN = 'unknown' + + class JobStatus: """Job status constants.""" PENDING = 'pending' @@ -19,12 +31,34 @@ class JobStatus: ALL = [PENDING, IN_PROGRESS, COMPLETED, FAILED, CANCELLED] +def validate_job_id(value) -> str: + """Validate a client-supplied job (prompt) id. + + Job ids must be UUIDs in the canonical lowercase hyphenated form. The id + is stored and compared verbatim everywhere downstream — history keys, + websocket events, and /interrupt matching — so accepting another spelling + would silently rewrite the client's id and then miss every exact-match + lookup. Rejecting loudly beats that. + + Returns the id unchanged. Raises ValueError when the value is not a + string in canonical UUID form. + """ + if not isinstance(value, str): + raise ValueError(f"job id must be a string, got {type(value).__name__}") + if str(uuid.UUID(value)) != value: + raise ValueError("job id must be a UUID in canonical lowercase hyphenated form") + return value + + # Media types that can be previewed in the frontend PREVIEWABLE_MEDIA_TYPES = frozenset({'images', 'video', 'audio', '3d', 'text'}) # 3D file extensions for preview fallback (no dedicated media_type exists) THREE_D_EXTENSIONS = frozenset({'.obj', '.fbx', '.gltf', '.glb', '.usdz'}) +# Text file extensions for preview fallback (the formats SaveText can produce) +TEXT_EXTENSIONS = frozenset({'.txt', '.md', '.json'}) + def has_3d_extension(filename: str) -> bool: lower = filename.lower() @@ -112,9 +146,10 @@ def is_previewable(media_type: str, item: dict) -> bool: Maintains backwards compatibility with existing logic. Priority: - 1. media_type is 'images', 'video', 'audio', or '3d' + 1. media_type is 'images', 'video', 'audio', '3d', or 'text' 2. format field starts with 'video/' or 'audio/' 3. filename has a 3D extension (.obj, .fbx, .gltf, .glb, .usdz) + 4. filename has a text extension (.txt, .md, .json, ...) """ if media_type in PREVIEWABLE_MEDIA_TYPES: return True @@ -125,10 +160,12 @@ def is_previewable(media_type: str, item: dict) -> bool: if fmt and (fmt.startswith('video/') or fmt.startswith('audio/')): return True - # Check for 3D files by extension + # Check for 3D and text files by extension filename = item.get('filename', '').lower() if any(filename.endswith(ext) for ext in THREE_D_EXTENSIONS): return True + if any(filename.endswith(ext) for ext in TEXT_EXTENSIONS): + return True return False @@ -224,6 +261,10 @@ def get_outputs_summary(outputs: dict) -> tuple[int, Optional[dict]]: Preview priority (matching frontend): 1. type="output" with previewable media 2. Any previewable media + + Text content entries (strings under 'text') are preview-only metadata, + matching the frontend's METADATA_KEYS: they can serve as the fallback + preview but are not counted as outputs. """ count = 0 preview_output = None @@ -244,7 +285,6 @@ def get_outputs_summary(outputs: dict) -> tuple[int, Optional[dict]]: if normalized is None: # Not a 3D file string — check for text preview if media_type == 'text': - count += 1 if preview_output is None: if isinstance(item, tuple): text_value = item[0] if item else '' @@ -387,3 +427,71 @@ def get_all_jobs( jobs = jobs[:limit] return (jobs, total_count) + + +def classify_job_for_cancel(prompt_id: str, running: list, queued: list, history: dict) -> str: + """Classify a job id for cancellation. + + Returns one of CANCEL_RUNNING, CANCEL_PENDING, CANCEL_TERMINAL, CANCEL_UNKNOWN. + + Queue items are tuples whose second element (index 1) is the prompt_id. + History is a dict keyed by prompt_id, so a job present there has already + finished and cancelling it is a no-op. + """ + for item in running: + if item[1] == prompt_id: + return CANCEL_RUNNING + for item in queued: + if item[1] == prompt_id: + return CANCEL_PENDING + if prompt_id in history: + return CANCEL_TERMINAL + return CANCEL_UNKNOWN + + +def cancel_job( + prompt_id: str, + running: list, + queued: list, + history: dict, + interrupt: Callable[[str], bool], + dequeue: Callable[[str], bool], +) -> str: + """Cancel a single job by id, regardless of state. + + Maps the cancel onto the runtime's existing mechanics: + - a running job is interrupted via ``interrupt`` + - a pending job is removed from the queue via ``dequeue`` + - a job that already finished (terminal) is a no-op + - an unknown id is a no-op (callers that need fail-fast behaviour should + validate ids up front with ``classify_job_for_cancel``) + + Both ``interrupt`` and ``dequeue`` take the prompt id and return whether + they acted on a job that was *actually* in that state, so the value returned + here reflects what truly happened rather than the (possibly stale) + classification. This matters around the narrow TOCTOU windows where a job + changes state between the caller's snapshot and the action: + + - a job classified RUNNING may have finished before ``interrupt`` fires: + ``interrupt`` returns False and this returns CANCEL_UNKNOWN (no-op). + - a job classified PENDING may have started executing before ``dequeue`` + fires: ``dequeue`` returns False, ``interrupt`` then catches the now- + running job and this returns CANCEL_RUNNING. If it had simply finished + instead, both return False and this returns CANCEL_UNKNOWN. + + ``interrupt`` must be atomic — interrupt the job only if it is still the one + running — so a cancel can never land on an unrelated prompt that started in + the meantime (see ``execution.PromptQueue.interrupt_if_running``). + """ + classification = classify_job_for_cancel(prompt_id, running, queued, history) + if classification == CANCEL_RUNNING: + return CANCEL_RUNNING if interrupt(prompt_id) else CANCEL_UNKNOWN + if classification == CANCEL_PENDING: + if dequeue(prompt_id): + return CANCEL_PENDING + # Left the pending queue between classification and dequeue: if it + # started executing, interrupt the now-running job; otherwise it has + # already finished and the cancel is a genuine no-op. + return CANCEL_RUNNING if interrupt(prompt_id) else CANCEL_UNKNOWN + # CANCEL_TERMINAL and CANCEL_UNKNOWN are intentional no-ops. + return classification diff --git a/comfy_extras/color_util.py b/comfy_extras/color_util.py new file mode 100644 index 000000000..d50795ae3 --- /dev/null +++ b/comfy_extras/color_util.py @@ -0,0 +1,23 @@ +def hex_to_rgb(value: str) -> tuple[int, int, int]: + h = value.lstrip("#") + if len(h) != 6: + return (255, 255, 255) + try: + return (int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)) + except ValueError: + return (255, 255, 255) + + +def readable_color(rgb: tuple[int, int, int]) -> tuple[int, int, int]: + r, g, b = rgb + lum = 0.299 * r + 0.587 * g + 0.114 * b + if lum >= 130: + return (r, g, b) + t = (130 - lum) / (255 - lum) + return (round(r + (255 - r) * t), round(g + (255 - g) * t), round(b + (255 - b) * t)) + + +def normalize_palette(colors) -> list[str]: + if isinstance(colors, dict): + colors = colors.values() + return [c.upper() for c in colors if isinstance(c, str) and c] diff --git a/comfy_extras/nodes_ace.py b/comfy_extras/nodes_ace.py index 044077b18..eaf234d5b 100644 --- a/comfy_extras/nodes_ace.py +++ b/comfy_extras/nodes_ace.py @@ -11,7 +11,7 @@ class TextEncodeAceStepAudio(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="TextEncodeAceStepAudio", - category="model/conditioning", + category="model/conditioning/ace", inputs=[ IO.Clip.Input("clip"), IO.String.Input("tags", multiline=True, dynamic_prompts=True), @@ -33,7 +33,7 @@ class TextEncodeAceStepAudio15(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="TextEncodeAceStepAudio1.5", - category="model/conditioning", + category="model/conditioning/ace", inputs=[ IO.Clip.Input("clip"), IO.String.Input("tags", multiline=True, dynamic_prompts=True), @@ -67,7 +67,7 @@ class EmptyAceStepLatentAudio(IO.ComfyNode): return IO.Schema( node_id="EmptyAceStepLatentAudio", display_name="Empty Ace Step 1.0 Latent Audio", - category="model/latent/audio", + category="model/latent/ace", inputs=[ IO.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.1), IO.Int.Input( @@ -90,7 +90,7 @@ class EmptyAceStep15LatentAudio(IO.ComfyNode): return IO.Schema( node_id="EmptyAceStep1.5LatentAudio", display_name="Empty Ace Step 1.5 Latent Audio", - category="model/latent/audio", + category="model/latent/ace", inputs=[ IO.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.01), IO.Int.Input( @@ -111,8 +111,8 @@ class ReferenceAudio(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="ReferenceTimbreAudio", - display_name="Reference Audio", - category="advanced/conditioning/audio", + display_name="Set Reference Audio", + category="model/conditioning", is_experimental=True, description="This node sets the reference audio for ace step 1.5", inputs=[ diff --git a/comfy_extras/nodes_apg.py b/comfy_extras/nodes_apg.py index 4a352038a..6e69b73f7 100644 --- a/comfy_extras/nodes_apg.py +++ b/comfy_extras/nodes_apg.py @@ -16,7 +16,7 @@ class APG(io.ComfyNode): return io.Schema( node_id="APG", display_name="Adaptive Projected Guidance", - category="model/sampling/custom_sampling", + category="model/sampling/custom", inputs=[ io.Model.Input("model"), io.Float.Input( diff --git a/comfy_extras/nodes_ar_video.py b/comfy_extras/nodes_ar_video.py index c22359eb2..9d8f64b20 100644 --- a/comfy_extras/nodes_ar_video.py +++ b/comfy_extras/nodes_ar_video.py @@ -19,7 +19,7 @@ class EmptyARVideoLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyARVideoLatent", - category="model/latent/video", + category="model/latent/autoregressive", inputs=[ io.Int.Input("width", default=832, min=16, max=8192, step=16), io.Int.Input("height", default=480, min=16, max=8192, step=16), @@ -85,7 +85,7 @@ class ARVideoI2V(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ARVideoI2V", - category="model/conditioning/video_models", + category="model/conditioning/autoregressive", inputs=[ io.Model.Input("model"), io.Vae.Input("vae"), diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index ff078f74c..4ac5ced53 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -16,7 +16,7 @@ class EmptyLatentAudio(IO.ComfyNode): return IO.Schema( node_id="EmptyLatentAudio", display_name="Empty Latent Audio", - category="model/latent/audio", + category="model/latent", essentials_category="Audio", inputs=[ IO.Float.Input("seconds", default=47.6, min=1.0, max=1000.0, step=0.1), @@ -41,7 +41,7 @@ class ConditioningStableAudio(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="ConditioningStableAudio", - category="model/conditioning", + category="model/conditioning/stable audio", inputs=[ IO.Conditioning.Input("positive"), IO.Conditioning.Input("negative"), @@ -70,7 +70,7 @@ class VAEEncodeAudio(IO.ComfyNode): node_id="VAEEncodeAudio", search_aliases=["audio to latent"], display_name="VAE Encode Audio", - category="model/latent/audio", + category="model/latent", inputs=[ IO.Audio.Input("audio"), IO.Vae.Input("vae"), @@ -115,7 +115,7 @@ class VAEDecodeAudio(IO.ComfyNode): node_id="VAEDecodeAudio", search_aliases=["latent to audio"], display_name="VAE Decode Audio", - category="model/latent/audio", + category="model/latent", inputs=[ IO.Latent.Input("samples"), IO.Vae.Input("vae"), @@ -137,7 +137,7 @@ class VAEDecodeAudioTiled(IO.ComfyNode): node_id="VAEDecodeAudioTiled", search_aliases=["latent to audio"], display_name="VAE Decode Audio (Tiled)", - category="model/latent/audio", + category="model/latent", inputs=[ IO.Latent.Input("samples"), IO.Vae.Input("vae"), @@ -158,7 +158,7 @@ class SaveAudio(IO.ComfyNode): return IO.Schema( node_id="SaveAudio", search_aliases=["export flac"], - display_name="Save Audio (FLAC)", + display_name="Save Audio (FLAC) (DEPRECATED)", category="audio", essentials_category="Audio", inputs=[ @@ -166,7 +166,9 @@ class SaveAudio(IO.ComfyNode): IO.String.Input("filename_prefix", default="audio/ComfyUI"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_deprecated=True, is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod @@ -174,11 +176,10 @@ class SaveAudio(IO.ComfyNode): if audio is None: raise ValueError("SaveAudio: input audio is None (source video may have no audio track).") return IO.NodeOutput( + audio, ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=format) ) - save_flac = execute # TODO: remove - class SaveAudioMP3(IO.ComfyNode): @classmethod @@ -186,7 +187,7 @@ class SaveAudioMP3(IO.ComfyNode): return IO.Schema( node_id="SaveAudioMP3", search_aliases=["export mp3"], - display_name="Save Audio (MP3)", + display_name="Save Audio (MP3) (DEPRECATED)", category="audio", essentials_category="Audio", inputs=[ @@ -195,7 +196,9 @@ class SaveAudioMP3(IO.ComfyNode): IO.Combo.Input("quality", options=["V0", "128k", "320k"], default="V0"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_deprecated=True, is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod @@ -203,13 +206,12 @@ class SaveAudioMP3(IO.ComfyNode): if audio is None: raise ValueError("SaveAudioMP3: input audio is None (source video may have no audio track).") return IO.NodeOutput( + audio, ui=UI.AudioSaveHelper.get_save_audio_ui( audio, filename_prefix=filename_prefix, cls=cls, format=format, quality=quality ) ) - save_mp3 = execute # TODO: remove - class SaveAudioOpus(IO.ComfyNode): @classmethod @@ -217,7 +219,7 @@ class SaveAudioOpus(IO.ComfyNode): return IO.Schema( node_id="SaveAudioOpus", search_aliases=["export opus"], - display_name="Save Audio (Opus)", + display_name="Save Audio (Opus) (DEPRECATED)", category="audio", inputs=[ IO.Audio.Input("audio"), @@ -225,7 +227,9 @@ class SaveAudioOpus(IO.ComfyNode): IO.Combo.Input("quality", options=["64k", "96k", "128k", "192k", "320k"], default="128k"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_deprecated=True, is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod @@ -233,12 +237,57 @@ class SaveAudioOpus(IO.ComfyNode): if audio is None: raise ValueError("SaveAudioOpus: input audio is None (source video may have no audio track).") return IO.NodeOutput( + audio, ui=UI.AudioSaveHelper.get_save_audio_ui( audio, filename_prefix=filename_prefix, cls=cls, format=format, quality=quality ) ) - save_opus = execute # TODO: remove + +class SaveAudioAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAudioAdvanced", + search_aliases=["save audio", "export audio", "output audio", "write audio", "flac", "mp3", "opus"], + display_name="Save Audio (Advanced)", + description="Saves the input audio to your ComfyUI output directory.", + category="audio", + inputs=[ + IO.Audio.Input("audio", tooltip="The audio to save."), + IO.String.Input( + "filename_prefix", + default="audio/ComfyUI", + tooltip=("The prefix for the file to save. May include formatting tokens such as %date:yyyy-MM-dd%."), + ), + IO.DynamicCombo.Input( + "format", + options=[ + IO.DynamicCombo.Option("flac", []), + IO.DynamicCombo.Option("mp3", [ + IO.Combo.Input("quality", options=["V0", "128k", "320k"], default="V0"), + ]), + IO.DynamicCombo.Option("opus", [ + IO.Combo.Input("quality", options=["64k", "96k", "128k", "192k", "320k"], default="128k"), + ]), + ], + tooltip="The file format in which to save the audio.", + ), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Audio.Output("audio")], + ) + + @classmethod + def execute(cls, audio, filename_prefix: str, format: dict) -> IO.NodeOutput: + file_format = format.get("format", None) + quality = format.get("quality", None) + if quality: + ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=file_format, quality=quality) + else: + ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=file_format) + return IO.NodeOutput(audio, ui=ui) class PreviewAudio(IO.ComfyNode): @@ -249,18 +298,20 @@ class PreviewAudio(IO.ComfyNode): search_aliases=["play audio"], display_name="Preview Audio", category="audio", + description="Preview the audio without saving it to the ComfyUI output directory.", inputs=[ IO.Audio.Input("audio"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod def execute(cls, audio) -> IO.NodeOutput: if audio is None: raise ValueError("PreviewAudio: input audio is None (source video may have no audio track).") - return IO.NodeOutput(ui=UI.PreviewAudio(audio, cls=cls)) + return IO.NodeOutput(audio, ui=UI.PreviewAudio(audio, cls=cls)) save_flac = execute # TODO: remove @@ -822,6 +873,7 @@ class AudioExtension(ComfyExtension): SaveAudio, SaveAudioMP3, SaveAudioOpus, + SaveAudioAdvanced, LoadAudio, PreviewAudio, ConditioningStableAudio, diff --git a/comfy_extras/nodes_bernini.py b/comfy_extras/nodes_bernini.py new file mode 100644 index 000000000..0537e0806 --- /dev/null +++ b/comfy_extras/nodes_bernini.py @@ -0,0 +1,108 @@ +import torch +from typing_extensions import override + +import comfy.model_management +import comfy.utils +import node_helpers +from comfy_api.latest import ComfyExtension, io + + +def _resize_long_edge(image, max_size, stride=16): + """Resize (preserve aspect) so the long edge <= max_size, then snap each side to `stride`""" + h, w = image.shape[1], image.shape[2] + scale = min(max_size / max(h, w), 1.0) + nh = max(stride, round(h * scale / stride) * stride) + nw = max(stride, round(w * scale / stride) * stride) + return comfy.utils.common_upscale(image[:, :, :, :3].movedim(-1, 1), nw, nh, "area", "disabled").movedim(1, -1) + + +class BerniniConditioning(io.ComfyNode): + """Bernini in-context conditioning for a Wan2.2-A14B model. + + Attaches the VAE-encoded source video / reference images to the conditioning + source video first, then each reference image + + The task is inferred from which inputs are connected: + (nothing) -> t2v (text-to-video) + source_video -> v2v (video-to-video) + source_video + ref_images -> rv2v (reference-guided video editing) + ref_images only -> r2v (reference-to-video) + source_video + ref_video -> ads2v (insert image/video into video) + + source_video is the edit base / canvas (resized to width x height). + reference_video is moving content to composite in. + Streams are ordered source_video, reference_video, then reference_images -> source_id (1, 2, 3, ...). + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="BerniniConditioning", + display_name="Bernini Conditioning", + category="model/conditioning/bernini", + description="Conditioning node for Bernini in-context video/image conditioning. It can be used for the following tasks: t2v (text-to-video), v2v (video-to-video), rv2v (reference-guided video editing), r2v (reference-to-video), ads2v (insert image/video into video)." + "Reference images injected as in-context tokens (r2v, rv2v) are encoded independently at their own native aspect ratio (long edge capped at ref_max_size)", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Int.Input("width", default=832, min=16, max=8192, step=16), + io.Int.Input("height", default=480, min=16, max=8192, step=16), + io.Int.Input("length", default=81, min=1, max=8192, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("source_video", optional=True, tooltip=("Source video to edit or restyle (v2v, rv2v). Resized to width/height and trimmed to length.")), + io.Image.Input("reference_video", optional=True, tooltip=("Video to insert into the source video (ads2v).")), + io.Autogrow.Input("reference_images", optional=True, + template=io.Autogrow.TemplatePrefix( + input=io.Image.Input("reference_image", tooltip=("Reference image injected as an in-context token (r2v, rv2v).")), + prefix="reference_image_", min=0, max=8)), + io.Int.Input("ref_max_size", default=848, min=16, max=8192, step=16, optional=True, tooltip=( + "Max size for the long edge of reference_video and reference_images. Resized with preserved aspect ratio and snapped to 16px.")), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, vae, width, height, length, batch_size, source_video=None, reference_video=None, reference_images=None, ref_max_size=848) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + + # source_video (1), reference_video (2), reference_images (3, 4, ...). + context = [] + if source_video is not None: + vid = comfy.utils.common_upscale(source_video[:length, :, :, :3].movedim(-1, 1), width, height, "area", "center").movedim(1, -1) + context.append(vae.encode(vid[:, :, :, :3])) + + if reference_video is not None: + ref_vid = _resize_long_edge(reference_video[:length], ref_max_size) # moving content, native aspect + context.append(vae.encode(ref_vid[:, :, :, :3])) + + # reference_images is an autogrow dict {reference_image_0: IMAGE, ...}; each slot is a + # separate stream at its own native aspect (a multi-image batch in one slot -> one stream per frame). + if reference_images: + for name in sorted(reference_images): + imgs = reference_images[name] + if imgs is None: + continue + for i in range(imgs.shape[0]): + img = _resize_long_edge(imgs[i:i + 1], ref_max_size) # native aspect per ref + context.append(vae.encode(img[:, :, :, :3])) + + if context: + positive = node_helpers.conditioning_set_values(positive, {"context_latents": context}) + negative = node_helpers.conditioning_set_values(negative, {"context_latents": context}) + + return io.NodeOutput(positive, negative, {"samples": latent}) + + +class BerniniExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [BerniniConditioning,] + + +async def comfy_entrypoint() -> BerniniExtension: + return BerniniExtension() diff --git a/comfy_extras/nodes_bg_removal.py b/comfy_extras/nodes_bg_removal.py index 9dc9ad854..c7b33a821 100644 --- a/comfy_extras/nodes_bg_removal.py +++ b/comfy_extras/nodes_bg_removal.py @@ -36,15 +36,15 @@ class RemoveBackground(IO.ComfyNode): category="image/background removal", description="Generates a foreground mask to remove the background from an image using a background removal model.", inputs=[ - IO.Image.Input("image", tooltip="Input image to remove the background from"), - IO.BackgroundRemoval.Input("bg_removal_model", tooltip="Background removal model used to generate the mask") + IO.BackgroundRemoval.Input("bg_removal_model", tooltip="Background removal model used to generate the mask"), + IO.Image.Input("image", tooltip="Input image to remove the background from") ], outputs=[ IO.Mask.Output("mask", tooltip="Generated foreground mask") ] ) @classmethod - def execute(cls, image, bg_removal_model): + def execute(cls, bg_removal_model, image): mask = bg_removal_model.encode_image(image) return IO.NodeOutput(mask) diff --git a/comfy_extras/nodes_boogu.py b/comfy_extras/nodes_boogu.py new file mode 100644 index 000000000..f3951c290 --- /dev/null +++ b/comfy_extras/nodes_boogu.py @@ -0,0 +1,97 @@ +import math + +import node_helpers +import comfy.utils +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class TextEncodeBooguEdit(io.ComfyNode): + """Boogu-Image Edit conditioning. + + The edit image is used twice, matching the reference pipeline: + - Qwen3-VL vision tokens (instruction understanding) -> positive only + - VAE reference latent (image identity) -> positive and negative + The ref latent is in both conds so it cancels under CFG (identity preserved); + the vision tokens are only in the positive so CFG amplifies the instruction. + The tokenizer selects the right system prompt automatically (image -> TI2I, + empty negative -> DROP), so no template plumbing is needed here. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TextEncodeBooguEdit", + category="model/conditioning/boogu", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.String.Input("negative_prompt", multiline=True, dynamic_prompts=True, advanced=True), + io.Vae.Input("vae"), + io.Autogrow.Input( + "images", + template=io.Autogrow.TemplateNames( + io.Image.Input("image"), + names=[f"image_{i}" for i in range(1, 17)], + min=0, + ), + tooltip="Reference image(s) to edit. Boogu focuses on one reference per sample; more are allowed.", + ), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, clip, prompt, negative_prompt, vae=None, images: io.Autogrow.Type = None) -> io.NodeOutput: + ref_latents = [] + images_vl = [] + + images = images or {} + for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])): + image = images[name] + if image is None: + continue + samples = image.movedim(-1, 1) + + # Vision tower input: the reference caps the VLM image at 384x384 + # (max_vlm_input_pil_pixels in pipeline_boogu.py). + total = int(384 * 384) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + width = round(samples.shape[3] * scale_by) + height = round(samples.shape[2] * scale_by) + s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") + images_vl.append(s.movedim(1, -1)[:, :, :, :3]) + + # Reference latent: align to 16 px (VAE /8 * patch_size 2). + if vae is not None: + total = int(1024 * 1024) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + width = round(samples.shape[3] * scale_by / 16.0) * 16 + height = round(samples.shape[2] * scale_by / 16.0) * 16 + s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") + ref_latents.append(vae.encode(s.movedim(1, -1)[:, :, :, :3])) + + # positive: instruction + vision tokens; negative: empty (no vision). Ref latent on both. + positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=images_vl)) + negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt)) + + if len(ref_latents) > 0: + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True) + + return io.NodeOutput(positive, negative) + + +class BooguExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + TextEncodeBooguEdit, + ] + + +async def comfy_entrypoint() -> BooguExtension: + return BooguExtension() diff --git a/comfy_extras/nodes_bounding_boxes.py b/comfy_extras/nodes_bounding_boxes.py new file mode 100644 index 000000000..de3709b91 --- /dev/null +++ b/comfy_extras/nodes_bounding_boxes.py @@ -0,0 +1,379 @@ +import json + +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageEnhance, ImageFont +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import hex_to_rgb, normalize_palette, readable_color + +_PREVIEW_LONG_EDGE = 1024 +_PREVIEW_DIM = 0.25 + + +def pixels_to_fractions(box: dict, width: int, height: int) -> dict: + w = width or 1 + h = height or 1 + return { + "x": box.get("x", 0) / w, + "y": box.get("y", 0) / h, + "w": box.get("width", 0) / w, + "h": box.get("height", 0) / h, + } + + +def fractions_to_pixels(box: dict, width: int, height: int) -> dict: + x, y = box.get("x", 0.0), box.get("y", 0.0) + w, h = box.get("w", 0.0), box.get("h", 0.0) + if w < 0: + x, w = x + w, -w + if h < 0: + y, h = y + h, -h + return { + "x": round(x * width), + "y": round(y * height), + "width": round(w * width), + "height": round(h * height), + } + + +def fractions_to_bbox_frame(boxes: list, width: int, height: int) -> list: + pixels = [ + fractions_to_pixels(box, width, height) + for box in boxes + if isinstance(box, dict) + ] + return [pixels] if pixels else [] + + +def _font(size: int): + try: + return ImageFont.load_default(size) + except Exception: + return ImageFont.load_default() + + +def _wrap(draw, text: str, font, max_w: float) -> list[str]: + lines = [] + for para in text.split("\n"): + line = "" + for word in para.split(): + test = word if not line else line + " " + word + if line and draw.textlength(test, font=font) > max_w: + lines.append(line) + line = word + else: + line = test + lines.append(line) + return lines + + +def _bg_from_image(image) -> Image.Image | None: + if image is None: + return None + try: + arr = (image[0].detach().cpu().numpy() * 255).clip(0, 255).astype(np.uint8) + return Image.fromarray(arr) + except Exception: + return None + + +def render_preview(regions, width, height, bg=None): + if bg is not None: + iw, ih = bg.size + long_edge = max(iw, ih) or 1 + scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge) + rw, rh = max(1, round(iw * scale)), max(1, round(ih * scale)) + base = bg.convert("RGB").resize((rw, rh), Image.LANCZOS) + base = ImageEnhance.Brightness(base).enhance(_PREVIEW_DIM) + img = base.convert("RGBA") + else: + long_edge = max(width, height) or 1 + scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge) + rw, rh = max(1, round(width * scale)), max(1, round(height * scale)) + grey = round(_PREVIEW_DIM * 128) + img = Image.new("RGBA", (rw, rh), (grey, grey, grey, 255)) + + overlay = Image.new("RGBA", (rw, rh), (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay) + fs = max(10, round(rh / 64)) + font = _font(fs) + tag_font = _font(max(9, fs - 2)) + line_h = fs + 2 + + for i, region in enumerate(regions): + if not isinstance(region, dict): + continue + palette = [c for c in (region.get("palette") or []) if c] + r, g, b = hex_to_rgb(palette[0]) if palette else (140, 140, 140) + x1 = max(0, min(rw, round(region.get("x", 0) * rw))) + y1 = max(0, min(rh, round(region.get("y", 0) * rh))) + x2 = max(0, min(rw, round((region.get("x", 0) + region.get("w", 0)) * rw))) + y2 = max(0, min(rh, round((region.get("y", 0) + region.get("h", 0)) * rh))) + if x2 < x1: + x1, x2 = x2, x1 + if y2 < y1: + y1, y2 = y2, y1 + + draw.rectangle([x1, y1, x2, y2], outline=(r, g, b, 255), width=2) + + swatches = palette[:5] + if swatches and (x2 - x1) > 2: + sh = max(5, fs // 2) + seg = (x2 - x1) / len(swatches) + for p, hexc in enumerate(swatches): + sx = x1 + round(p * seg) + draw.rectangle([sx, y1, x1 + round((p + 1) * seg), y1 + sh], fill=hex_to_rgb(hexc)) + + etype = "text" if region.get("type") == "text" else "obj" + tag = str(i + 1).zfill(2) + tw = draw.textlength(tag, font=tag_font) + draw.rectangle([x1, y1, x1 + tw + 6, y1 + fs + 2], fill=(r, g, b, 255)) + tag_fill = (0, 0, 0, 255) if (0.299 * r + 0.587 * g + 0.114 * b) > 140 else (255, 255, 255, 255) + draw.text((x1 + 3, y1 + 1), tag, fill=tag_fill, font=tag_font) + + body = region.get("desc", "") or "" + if etype == "text" and region.get("text"): + body = '"%s"%s' % (region["text"], " — " + body if body else "") + if body and (x2 - x1) > 8: + ty = y1 + fs + 5 + for line in _wrap(draw, body, font, x2 - x1 - 8): + if ty > y2: + break + draw.text((x1 + 4, ty), line, fill=readable_color((r, g, b)) + (255,), font=font) + ty += line_h + + composed = Image.alpha_composite(img, overlay).convert("RGB") + arr = np.asarray(composed, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + +def boxes_to_regions(boxes, width: int, height: int) -> list: + regions: list = [] + if not isinstance(boxes, list): + return regions + for box in boxes: + if not isinstance(box, dict): + continue + meta = box.get("metadata") + meta = meta if isinstance(meta, dict) else {} + regions.append({ + **pixels_to_fractions(box, width, height), + "type": meta.get("type", "obj"), + "text": meta.get("text", ""), + "desc": meta.get("desc", ""), + "palette": meta.get("palette", []), + }) + return regions + + +def normalize_incoming_boxes(bboxes) -> list: + if isinstance(bboxes, dict): + frame = [bboxes] + elif not isinstance(bboxes, list) or not bboxes: + frame = [] + elif isinstance(bboxes[0], dict): + frame = bboxes + else: + frame = bboxes[0] if isinstance(bboxes[0], list) else [] + boxes = [] + for box in frame: + if not isinstance(box, dict): + continue + norm = { + "x": box.get("x", 0), + "y": box.get("y", 0), + "width": box.get("width", 0), + "height": box.get("height", 0), + } + meta = box.get("metadata") + if isinstance(meta, dict): + norm["metadata"] = meta + boxes.append(norm) + return boxes + + +def _looks_like_element(box: dict) -> bool: + bbox = box.get("bbox") + return isinstance(bbox, (list, tuple)) and len(bbox) == 4 + + +def _looks_like_bbox(box: dict) -> bool: + return all(key in box for key in ("x", "y", "width", "height")) + + +def elements_to_boxes(elements: list, width: int, height: int) -> list: + boxes = [] + for element in elements: + if not isinstance(element, dict): + continue + bbox = element.get("bbox") + if not (isinstance(bbox, (list, tuple)) and len(bbox) == 4): + raise ValueError("bboxes element is missing a valid 'bbox' [ymin, xmin, ymax, xmax]") + try: + ymin, xmin, ymax, xmax = (float(v) / 1000.0 for v in bbox) + except (TypeError, ValueError): + raise ValueError("bboxes element 'bbox' must contain four numbers") + etype = "text" if element.get("type") == "text" else "obj" + boxes.append({ + "x": round(min(xmin, xmax) * width), + "y": round(min(ymin, ymax) * height), + "width": round(abs(xmax - xmin) * width), + "height": round(abs(ymax - ymin) * height), + "metadata": { + "type": etype, + "text": element.get("text", "") if etype == "text" else "", + "desc": element.get("desc", ""), + "palette": element.get("color_palette", []) or [], + }, + }) + return boxes + + +def boxes_from_input(data, width: int, height: int) -> list: + if data is None: + return [] + if isinstance(data, str): + text = data.strip() + if not text: + return [] + try: + data = json.loads(text) + except (ValueError, TypeError) as exc: + raise ValueError(f"bboxes string input is not valid JSON: {exc}") from exc + if isinstance(data, dict): + if _looks_like_element(data): + return elements_to_boxes([data], width, height) + if _looks_like_bbox(data): + return normalize_incoming_boxes(data) + raise ValueError( + "bboxes dict must be a bounding box (x, y, width, height) or an element (with a 'bbox')" + ) + if not isinstance(data, list): + raise ValueError( + "bboxes input must be bounding boxes, elements, or a JSON string, " + f"got {type(data).__name__}" + ) + if not data: + return [] + first = data[0] + if isinstance(first, list): + return normalize_incoming_boxes(data) + if isinstance(first, dict): + if _looks_like_element(first): + return elements_to_boxes(data, width, height) + if _looks_like_bbox(first): + return normalize_incoming_boxes(data) + raise ValueError( + "bboxes items must be bounding boxes (x, y, width, height) or elements (with a 'bbox')" + ) + raise ValueError( + f"bboxes list must contain bounding boxes or elements, got {type(first).__name__}" + ) + + +def _norm_bbox(region: dict) -> list[int]: + def grid(value: float) -> int: + return max(0, min(1000, round(value * 1000))) + + x, y = region.get("x", 0.0), region.get("y", 0.0) + w, h = region.get("w", 0.0), region.get("h", 0.0) + ymin, xmin, ymax, xmax = grid(y), grid(x), grid(y + h), grid(x + w) + if ymin > ymax: + ymin, ymax = ymax, ymin + if xmin > xmax: + xmin, xmax = xmax, xmin + return [ymin, xmin, ymax, xmax] + + +def build_elements(regions: list) -> list: + elements = [] + for region in regions: + if not isinstance(region, dict): + continue + etype = "text" if region.get("type") == "text" else "obj" + element = {"type": etype} + element["bbox"] = _norm_bbox(region) + if etype == "text": + element["text"] = region.get("text", "") + element["desc"] = region.get("desc", "") + palette = normalize_palette(region.get("palette", [])) + if palette: + element["color_palette"] = palette[:5] + elements.append(element) + return elements + + +class CreateBoundingBoxes(io.ComfyNode): + @classmethod + def define_schema(cls): + editor_state = io.BoundingBoxes.Input( + "editor_state", + socketless=False, + tooltip="Draw bounding boxes and set each box type, text, description, color palette. Start with background element first and foreground last.", + ) + return io.Schema( + node_id="CreateBoundingBoxes", + display_name="Create Bounding Boxes", + category="utilities", + description="Draw bounding boxes in a canvas. Outputs Ideogram prompt elements, pixel-space bounding boxes, and a preview image.", + inputs=[ + io.Image.Input( + "background", + optional=True, + tooltip="Optional image used as background in the canvas and preview.", + ), + io.MultiType.Input( + "bboxes", + [io.BoundingBox, io.Array, io.String], + optional=True, + tooltip="Bounding boxes, elements, or a JSON string to initialize the canvas. A new upstream value initializes the canvas; edits made on the canvas take priority and are kept until the upstream value changes again.", + ), + io.Int.Input("width", default=1024, min=64, max=16384, step=16, + tooltip="Width of the canvas and the pixel grid for the bounding boxes."), + io.Int.Input("height", default=1024, min=64, max=16384, step=16, + tooltip="Height of the canvas and the pixel grid for the bounding boxes."), + editor_state, + io.BoundingBoxes.Input( + "last_incoming", + optional=True, + tooltip="Internal state managed by the canvas: the upstream bboxes value that last initialized it. Leave empty to re-initialize the canvas from the bboxes input on the next run.", + ), + ], + outputs=[ + io.Image.Output(display_name="preview"), + io.BoundingBox.Output(display_name="bboxes"), + io.Array.Output(display_name="elements"), + ], + is_output_node=True, + is_experimental=True, + ) + + @classmethod + def execute(cls, width, height, editor_state=None, last_incoming=None, background=None, bboxes=None) -> io.NodeOutput: + incoming = boxes_from_input(bboxes, width, height) + applied = last_incoming if isinstance(last_incoming, list) else [] + upstream_changed = bool(incoming) and incoming != applied + source = incoming if upstream_changed else (editor_state or []) + regions = boxes_to_regions(source, width, height) + preview = render_preview(regions, width, height, _bg_from_image(background)) + ui = {"dims": [width, height]} + if incoming: + ui["input_bboxes"] = incoming + return io.NodeOutput( + preview, + fractions_to_bbox_frame(regions, width, height), + build_elements(regions), + ui=ui, + ) + + +class BoundingBoxesExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [CreateBoundingBoxes] + + +async def comfy_entrypoint() -> BoundingBoxesExtension: + return BoundingBoxesExtension() diff --git a/comfy_extras/nodes_camera_trajectory.py b/comfy_extras/nodes_camera_trajectory.py index 13a1448f4..280d136af 100644 --- a/comfy_extras/nodes_camera_trajectory.py +++ b/comfy_extras/nodes_camera_trajectory.py @@ -153,7 +153,7 @@ class WanCameraEmbedding(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanCameraEmbedding", - category="model/conditioning/video_models", + category="model/conditioning/wan/camera", inputs=[ io.Combo.Input( "camera_pose", diff --git a/comfy_extras/nodes_chroma_radiance.py b/comfy_extras/nodes_chroma_radiance.py index a4f673001..059344f3c 100644 --- a/comfy_extras/nodes_chroma_radiance.py +++ b/comfy_extras/nodes_chroma_radiance.py @@ -13,7 +13,7 @@ class EmptyChromaRadianceLatentImage(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="EmptyChromaRadianceLatentImage", - category="model/latent/chroma_radiance", + category="model/latent/chroma radiance", inputs=[ io.Int.Input(id="width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input(id="height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -33,7 +33,7 @@ class ChromaRadianceOptions(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="ChromaRadianceOptions", - category="model/patch/chroma_radiance", + category="model/patch/chroma radiance", description="Allows setting advanced options for the Chroma Radiance model.", inputs=[ io.Model.Input(id="model"), diff --git a/comfy_extras/nodes_clip_sdxl.py b/comfy_extras/nodes_clip_sdxl.py index 7a001af6f..08fbbd827 100644 --- a/comfy_extras/nodes_clip_sdxl.py +++ b/comfy_extras/nodes_clip_sdxl.py @@ -9,7 +9,8 @@ class CLIPTextEncodeSDXLRefiner(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeSDXLRefiner", - category="advanced/conditioning", + display_name="CLIP Text Encode (SDXL Refiner)", + category="model/conditioning/stable diffusion", inputs=[ io.Float.Input("ascore", default=6.0, min=0.0, max=1000.0, step=0.01), io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), @@ -30,7 +31,8 @@ class CLIPTextEncodeSDXL(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeSDXL", - category="advanced/conditioning", + display_name="CLIP Text Encode (SDXL)", + category="model/conditioning/stable diffusion", inputs=[ io.Clip.Input("clip"), io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), diff --git a/comfy_extras/nodes_color.py b/comfy_extras/nodes_color.py index 01a05035e..6d10b26f4 100644 --- a/comfy_extras/nodes_color.py +++ b/comfy_extras/nodes_color.py @@ -1,5 +1,6 @@ from typing_extensions import override from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import hex_to_rgb class ColorToRGBInt(io.ComfyNode): @@ -7,29 +8,38 @@ class ColorToRGBInt(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="ColorToRGBInt", - display_name="Color to RGB Int", + display_name="Color Picker", category="utilities", - description="Convert a color to a RGB integer value.", + description="Return a color RGB integer value and hexadecimal representation.", inputs=[ io.Color.Input("color"), ], outputs=[ io.Int.Output(display_name="rgb_int"), + io.Color.Output(display_name="hex"), + io.Float.Output(display_name="alpha"), ], ) @classmethod - def execute( - cls, - color: str, - ) -> io.NodeOutput: - # expect format #RRGGBB - if len(color) != 7 or color[0] != "#": - raise ValueError("Color must be in format #RRGGBB") - r = int(color[1:3], 16) - g = int(color[3:5], 16) - b = int(color[5:7], 16) - return io.NodeOutput(r * 256 * 256 + g * 256 + b) + def execute(cls, color: str) -> io.NodeOutput: + # expect format #RRGGBB or #RRGGBBAA + if len(color) not in (7, 9) or color[0] != "#": + raise ValueError("Color must be in format #RRGGBB or #RRGGBBAA") + try: + int(color[1:], 16) + except ValueError: + raise ValueError("Color must be in format #RRGGBB or #RRGGBBAA") from None + + alpha = 1.0 + if len(color) == 9: + alpha = int(color[7:9], 16) / 255.0 + color = color[:7] + + r, g, b = hex_to_rgb(color) + + rgb_int = r * 256 * 256 + g * 256 + b + return io.NodeOutput(rgb_int, color, alpha) class ColorExtension(ComfyExtension): diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py index b745a43af..c8091b7a4 100644 --- a/comfy_extras/nodes_cond.py +++ b/comfy_extras/nodes_cond.py @@ -8,7 +8,8 @@ class CLIPTextEncodeControlnet(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="CLIPTextEncodeControlnet", - category="experimental/conditioning", + display_name="CLIP Text Encode (Controlnet)", + category="model/conditioning", inputs=[ io.Clip.Input("clip"), io.Conditioning.Input("conditioning"), @@ -35,11 +36,12 @@ class T5TokenizerOptions(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="T5TokenizerOptions", - category="experimental/conditioning", + display_name="T5 Tokenizer Options", + category="model/conditioning", inputs=[ io.Clip.Input("clip"), - io.Int.Input("min_padding", default=0, min=0, max=10000, step=1, advanced=True), - io.Int.Input("min_length", default=0, min=0, max=10000, step=1, advanced=True), + io.Int.Input("min_padding", default=0, min=0, max=10000, step=1), + io.Int.Input("min_length", default=0, min=0, max=10000, step=1), ], outputs=[io.Clip.Output()], is_experimental=True, diff --git a/comfy_extras/nodes_context_windows.py b/comfy_extras/nodes_context_windows.py index d9e32b9d9..15d2dc506 100644 --- a/comfy_extras/nodes_context_windows.py +++ b/comfy_extras/nodes_context_windows.py @@ -13,21 +13,22 @@ class ContextWindowsManualNode(io.ComfyNode): description="Manually set context windows.", inputs=[ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), - io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window.", advanced=True), - io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window.", advanced=True), + io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window."), + io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window."), io.Combo.Input("context_schedule", options=[ comfy.context_windows.ContextSchedules.STATIC_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, comfy.context_windows.ContextSchedules.BATCHED, - ], tooltip="The stride of the context window."), - io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), + ], default=comfy.context_windows.ContextSchedules.STATIC_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."), io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."), io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), io.Int.Input("dim", min=0, max=5, default=0, tooltip="The dimension to apply the context windows to."), io.Boolean.Input("freenoise", default=False, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending."), - io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window, for example setting this to '0' will use the initial start image for each window."), + io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window. For concat-style I2V models (e.g. Wan I2V, HunyuanVideo I2V, Cosmos I2V, SVD) the encoded start image lives in the c_concat conditioning channels; setting this to '0' will retain that start image content at sub-pos 0 of every window."), io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index."), + io.String.Input("latent_retain_index_list", default="", tooltip="List of latent indices to retain in the noise latent itself for each window. Use for workflows where reference content (e.g. a start image) lives directly in the noise latent rather than in separate conditioning channels (e.g. inplace-style I2V like LTXV, AnimateDiff). Independent of cond_retain_index_list."), io.Boolean.Input("causal_window_fix", default=True, tooltip="Whether to add a causal fix frame to non-0-indexed context windows."), ], outputs=[ @@ -38,7 +39,7 @@ class ContextWindowsManualNode(io.ComfyNode): @classmethod def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, dim: int, freenoise: bool, - cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, causal_window_fix: bool=True) -> io.Model: + cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, latent_retain_index_list: list[int]=[], causal_window_fix: bool=True) -> io.Model: model = model.clone() model.model_options["context_handler"] = comfy.context_windows.IndexListContextHandler( context_schedule=comfy.context_windows.get_matching_context_schedule(context_schedule), @@ -51,6 +52,7 @@ class ContextWindowsManualNode(io.ComfyNode): freenoise=freenoise, cond_retain_index_list=cond_retain_index_list, split_conds_to_windows=split_conds_to_windows, + latent_retain_index_list=latent_retain_index_list, causal_window_fix=causal_window_fix, ) # make memory usage calculation only take into account the context window latents @@ -65,32 +67,71 @@ class WanContextWindowsManualNode(ContextWindowsManualNode): schema = super().define_schema() schema.node_id = "WanContextWindowsManual" schema.display_name = "WAN Context Windows (Manual)" - schema.description = "Manually set context windows for WAN-like models (dim=2)." + schema.display_name = "Wan Context Windows" + schema.description = "Set context windows for Wan-like models." + schema.category="model/patch/wan" schema.inputs = [ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), - io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window.", advanced=True), - io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window.", advanced=True), + io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window in real frames. Must be 4*n + 1."), + io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window in real frames."), io.Combo.Input("context_schedule", options=[ comfy.context_windows.ContextSchedules.STATIC_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, comfy.context_windows.ContextSchedules.BATCHED, - ], tooltip="The stride of the context window."), + ], default=comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), - io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules.", advanced=True), io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), - io.Boolean.Input("freenoise", default=False, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending."), - #io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window, for example setting this to '0' will use the initial start image for each window."), - #io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index."), + io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True), + io.Boolean.Input("retain_first_frame", default=False, tooltip="Retain the first I2V frame in every context window (may help retain initial reference)."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True), ] return schema @classmethod def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, freenoise: bool, - cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False) -> io.Model: - context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1 - context_overlap = max(((context_overlap - 1) // 4) + 1, 0) # at least overlap 0 - return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, cond_retain_index_list=cond_retain_index_list, split_conds_to_windows=split_conds_to_windows) + retain_first_frame: bool=False, split_conds_to_windows: bool=False) -> io.Model: + context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1 + context_overlap = max(context_overlap // 4, 0) # at least overlap 0 + retain_index_list = "0" if retain_first_frame else "" + return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, cond_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows) + + +class LTXVContextWindowsNode(ContextWindowsManualNode): + @classmethod + def define_schema(cls) -> io.Schema: + schema = super().define_schema() + schema.node_id = "LTXVContextWindows" + schema.display_name = "LTXV Context Windows" + schema.description = "Set context windows for LTXV-like models." + schema.inputs = [ + io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), + io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=8, default=145, tooltip="The length of the context window in real frames. Must be 8*n + 1."), + io.Int.Input("context_overlap", min=0, step=8, default=40, tooltip="The overlap of the context window in real frames."), + io.Combo.Input("context_schedule", options=[ + comfy.context_windows.ContextSchedules.STATIC_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, + comfy.context_windows.ContextSchedules.BATCHED, + ], default=comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules.", advanced=True), + io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), + io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True), + io.Boolean.Input("retain_first_frame", default=False, tooltip="Retain the first latent frame in every context window (may help retain initial reference)."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True), + ] + return schema + + @classmethod + def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, fuse_method: str, freenoise: bool, + retain_first_frame: bool=False, split_conds_to_windows: bool=False, context_stride: int=1, closed_loop: bool=False) -> io.Model: + context_length = max(((context_length - 1) // 8) + 1, 1) # at least length 1 + context_overlap = max(context_overlap // 8, 0) # at least overlap 0 + retain_index_list = "0" if retain_first_frame else "" + return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, + cond_retain_index_list=retain_index_list, latent_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows) class ContextWindowsExtension(ComfyExtension): @@ -98,6 +139,7 @@ class ContextWindowsExtension(ComfyExtension): return [ ContextWindowsManualNode, WanContextWindowsManualNode, + LTXVContextWindowsNode, ] def comfy_entrypoint(): diff --git a/comfy_extras/nodes_controlnet.py b/comfy_extras/nodes_controlnet.py index 17d965405..eb476f497 100644 --- a/comfy_extras/nodes_controlnet.py +++ b/comfy_extras/nodes_controlnet.py @@ -9,6 +9,8 @@ class SetUnionControlNetType(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SetUnionControlNetType", + search_aliases=["set controlnet type", "union controlnet type"], + display_name="Set Union ControlNet Type", category="model/conditioning/controlnet", inputs=[ io.ControlNet.Input("control_net"), @@ -39,6 +41,7 @@ class ControlNetInpaintingAliMamaApply(io.ComfyNode): return io.Schema( node_id="ControlNetInpaintingAliMamaApply", search_aliases=["masked controlnet"], + display_name="Apply ControlNet Inpainting (AliMama)", category="model/conditioning/controlnet", inputs=[ io.Conditioning.Input("positive"), diff --git a/comfy_extras/nodes_cosmos.py b/comfy_extras/nodes_cosmos.py index d754ab442..93cc67a6c 100644 --- a/comfy_extras/nodes_cosmos.py +++ b/comfy_extras/nodes_cosmos.py @@ -13,7 +13,7 @@ class EmptyCosmosLatentVideo(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="EmptyCosmosLatentVideo", - category="model/latent/video", + category="model/latent/cosmos", inputs=[ io.Int.Input("width", default=1280, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=704, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -45,7 +45,7 @@ class CosmosImageToVideoLatent(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="CosmosImageToVideoLatent", - category="model/conditioning/inpaint", + category="model/conditioning/cosmos", inputs=[ io.Vae.Input("vae"), io.Int.Input("width", default=1280, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -88,7 +88,7 @@ class CosmosPredict2ImageToVideoLatent(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="CosmosPredict2ImageToVideoLatent", - category="model/conditioning/inpaint", + category="model/conditioning/cosmos", inputs=[ io.Vae.Input("vae"), io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index c3346bf09..56ef5f526 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -1,5 +1,7 @@ import math import comfy.samplers +import comfy.sampler_helpers +import comfy.patcher_extension import comfy.sample from comfy.k_diffusion import sampling as k_diffusion_sampling from comfy.k_diffusion import sa_solver @@ -727,7 +729,7 @@ class SamplerCustom(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SamplerCustom", - category="model/sampling/custom_sampling", + category="model/sampling/custom", inputs=[ io.Model.Input("model"), io.Boolean.Input("add_noise", default=True, advanced=True), @@ -894,6 +896,85 @@ class DualCFGGuider(io.ComfyNode): get_guider = execute +class Guider_DualModel(comfy.samplers.CFGGuider): + # Runs the positive (cond) pass on the main model and the negative (uncond) pass on a separate model + def __init__(self, model_patcher, uncond_model_patcher): + super().__init__(model_patcher) + self.uncond_model_patcher = uncond_model_patcher + self.uncond_inner = None + + def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, latent_shapes=None): + self.uncond_inner = None + self.uncond_loaded = [] + self._uncond_neg = None + # skip at cfg 1.0 + if not math.isclose(self.cfg, 1.0): + uc = {"negative": list(map(lambda a: a.copy(), self.conds["negative"]))} + self.uncond_inner, uc, self.uncond_loaded = comfy.sampler_helpers.prepare_sampling( + self.uncond_model_patcher, noise.shape, uc, self.uncond_model_patcher.model_options) + self._uncond_neg = uc["negative"] + self.uncond_model_patcher.pre_run() + try: + return super().outer_sample(noise, latent_image, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=latent_shapes) + finally: + if self.uncond_inner is not None: + self.uncond_model_patcher.cleanup() + comfy.sampler_helpers.cleanup_models({"negative": self._uncond_neg}, self.uncond_loaded) + self.uncond_inner = None + + def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=None): + if self.uncond_inner is not None: + li = latent_image + if li is not None and torch.count_nonzero(li) > 0: + li = self.uncond_inner.process_latent_in(li) + self._uncond_conds = comfy.samplers.process_conds( + self.uncond_inner, noise, {"negative": self._uncond_neg}, device, li, denoise_mask, seed, latent_shapes=latent_shapes)["negative"] + return super().inner_sample(noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=latent_shapes) + + def predict_noise(self, x, timestep, model_options={}, seed=None): + positive = self.conds.get("positive", None) + cond = comfy.samplers.calc_cond_batch(self.inner_model, [positive], x, timestep, model_options)[0] + # uncond model not loaded (base cfg==1/no negative), or cfg driven to 1.0 this step -> single model, cond only + if self.uncond_inner is None or (math.isclose(self.cfg, 1.0) and not model_options.get("disable_cfg1_optimization", False)): + return cond + + uncond_model_options = model_options + if "multigpu_clones" in model_options: # TODO: support multigpu instead of just running uncond on a single GPU + uncond_model_options = {k: v for k, v in model_options.items() if k != "multigpu_clones"} + uncond = comfy.samplers.calc_cond_batch(self.uncond_inner, [self._uncond_conds], x, timestep, uncond_model_options)[0] + return comfy.samplers.cfg_function(self.inner_model, cond, uncond, self.cfg, x, timestep, + model_options=model_options, cond=positive, uncond=self._uncond_conds) + +class DualModelGuider(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DualModelGuider", + display_name="Dual Model CFG Guider", + category="model/sampling/guiders", + is_experimental=True, + inputs=[ + io.Model.Input("model", tooltip="Model used for the positive (conditional) pass."), + io.Model.Input("model_negative", optional=True, tooltip="Model used for the negative (unconditional) pass. Use the same model for ordinary CFG."), + io.Conditioning.Input("positive"), + io.Float.Input("cfg", default=4.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Conditioning.Input("negative", optional=True, tooltip="Negative conditioning run on the negative model. Leave unconnected for a text-free (image-only) unconditional pass."), + ], + outputs=[io.Guider.Output()], + ) + + @classmethod + def execute(cls, model, positive, cfg, model_negative=None, negative=None) -> io.NodeOutput: + if negative is None: + negative = [[None, {}]] # null cond -> no cross_attn -> model runs image-only + + guider = Guider_DualModel(model, model_negative) if model_negative is not None else comfy.samplers.CFGGuider(model) + guider.set_conds(positive, negative) + guider.set_cfg(cfg) + return io.NodeOutput(guider) + + get_guider = execute + class DisableNoise(io.ComfyNode): @classmethod def define_schema(cls): @@ -934,7 +1015,7 @@ class SamplerCustomAdvanced(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SamplerCustomAdvanced", - category="model/sampling/custom_sampling", + category="model/sampling/custom", inputs=[ io.Noise.Input("noise"), io.Guider.Input("guider"), @@ -989,7 +1070,7 @@ class AddNoise(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="AddNoise", - category="experimental/custom_sampling/noise", + category="model/sampling/noise", is_experimental=True, inputs=[ io.Model.Input("model"), @@ -1039,7 +1120,7 @@ class ManualSigmas(io.ComfyNode): return io.Schema( node_id="ManualSigmas", search_aliases=["custom noise schedule", "define sigmas"], - category="experimental/custom_sampling", + category="model/sampling/sigmas", is_experimental=True, inputs=[ io.String.Input("sigmas", default="1, 0.5", multiline=False) @@ -1054,11 +1135,53 @@ class ManualSigmas(io.ComfyNode): sigmas = torch.FloatTensor(sigmas) return io.NodeOutput(sigmas) +class CFGOverride(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CFGOverride", + display_name="CFG Override", + description="Override cfg to a fixed value over a [start, end] percent (sigma) range. " + "With multiple overrides, the one nearest the sampler wins on overlap.", + category="model/sampling/guiders", + inputs=[ + io.Model.Input("model"), + io.Float.Input("cfg", default=1.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001), + io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001), + ], + outputs=[io.Model.Output()], + ) + + @classmethod + def execute(cls, model, cfg, start_percent, end_percent) -> io.NodeOutput: + ms = model.get_model_object("model_sampling") + sigma_hi = ms.percent_to_sigma(start_percent) # percent->sigma decreasing, so hi >= lo + sigma_lo = ms.percent_to_sigma(end_percent) + + def predict_noise_wrapper(executor, *args, **kwargs): + sigma = float(args[1].flatten()[0]) # args = (x, timestep, model_options, seed) + if not (sigma_lo <= sigma <= sigma_hi): + return executor(*args, **kwargs) + guider = executor.class_obj # guider.cfg feeds cond_scale + saved = guider.cfg + guider.cfg = cfg + try: + return executor(*args, **kwargs) + finally: + guider.cfg = saved # restore for other steps/overrides + + m = model.clone() + m.add_wrapper(comfy.patcher_extension.WrappersMP.PREDICT_NOISE, predict_noise_wrapper) + return io.NodeOutput(m) + + class CustomSamplersExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: return [ SamplerCustom, + CFGOverride, BasicScheduler, KarrasScheduler, ExponentialScheduler, @@ -1087,6 +1210,7 @@ class CustomSamplersExtension(ComfyExtension): SamplingPercentToSigma, CFGGuider, DualCFGGuider, + DualModelGuider, BasicGuider, RandomNoise, DisableNoise, diff --git a/comfy_extras/nodes_dataset.py b/comfy_extras/nodes_dataset.py index 104d16d91..73fe75b7f 100644 --- a/comfy_extras/nodes_dataset.py +++ b/comfy_extras/nodes_dataset.py @@ -411,6 +411,21 @@ class ImageProcessingNode(io.ComfyNode): return has_group + @classmethod + def _ensure_image_list(cls, images): + """Normalize to a flat list of [1, H, W, C] tensors.""" + if isinstance(images, torch.Tensor): + if images.ndim != 4: + raise ValueError(f"Expected 4D image tensor, got shape {tuple(images.shape)}") + return [images[i:i+1] for i in range(images.shape[0])] + + flat = [] + for item in images: + if not isinstance(item, torch.Tensor) or item.ndim != 4: + raise ValueError(f"Expected 4D image tensor, got {type(item).__name__} shape {getattr(item, 'shape', None)}") + flat.extend([item[i:i+1] for i in range(item.shape[0])]) + return flat + @classmethod def define_schema(cls): if cls.node_id is None: @@ -458,6 +473,9 @@ class ImageProcessingNode(io.ComfyNode): """Execute the node. Routes to _process or _group_process based on mode.""" is_group = cls._detect_processing_mode() + if is_group: + images = cls._ensure_image_list(images) + # Extract scalar values from lists for parameters params = {} for k, v in kwargs.items(): @@ -1565,7 +1583,7 @@ class LoadTrainingDataset(io.ComfyNode): shard_path = os.path.join(dataset_dir, shard_file) with open(shard_path, "rb") as f: - shard_data = torch.load(f) + shard_data = torch.load(f, weights_only=True) all_latents.extend(shard_data["latents"]) all_conditioning.extend(shard_data["conditioning"]) diff --git a/comfy_extras/nodes_depth_anything_3.py b/comfy_extras/nodes_depth_anything_3.py new file mode 100644 index 000000000..020112515 --- /dev/null +++ b/comfy_extras/nodes_depth_anything_3.py @@ -0,0 +1,681 @@ +"""ComfyUI nodes for Depth Anything 3. +Model capability matrix: + +Variant head_type has_sky has_conf cam_dec +DA3-Small dualdpt False True yes +DA3-Base dualdpt False True yes +DA3-Mono-Large dpt True False no +DA3-Metric-Large dpt True False no (raw output is metres) +""" + +from __future__ import annotations + +import logging +from typing_extensions import override + +import torch + +import comfy.model_management as mm +import comfy.sd +import folder_paths +from comfy.ldm.colormap import turbo as _turbo +from comfy.ldm.depth_anything_3 import preprocess as da3_preprocess +from comfy_api.latest import ComfyExtension, Types, io +from comfy.ldm.moge.geometry import triangulate_grid_mesh + +DA3ModelType = io.Custom("DA3_MODEL") +DA3Geometry = io.Custom("DA3_GEOMETRY") +DA3PointCloud = io.Custom("DA3_POINT_CLOUD") + +# DA3_GEOMETRY is a dict with these optional keys (absent when the upstream model didn't produce them): +# +# Per-frame tensors - B = batch size in mono mode; B = S (number of views) in multi-view mode. +# "depth": torch.Tensor (B, H, W) -- raw model depth (always present; matches MoGe convention) +# "image": torch.Tensor (B, H, W, 3) -- source image in [0, 1], CPU (always present) +# "mode": str -- "mono" or "multiview" (always present) +# "sky": torch.Tensor (B, H, W) -- sky probability in [0, 1] (Mono/Metric variants only) +# "confidence": torch.Tensor (B, H, W) -- raw model confidence output (Small/Base variants only) +# +# Multi-view only - S = number of views; the leading 1 is the scene dimension from the model. +# "extrinsics": torch.Tensor (1, S, 3, 4) -- world-to-camera [R|t] matrices +# "intrinsics": torch.Tensor (1, S, 3, 3) -- pixel-space intrinsics +# +# DA3_POINT_CLOUD is a dict: +# "points": torch.Tensor (N, 3) -- 3-D coords in glTF convention (Y-up, Z-back) +# "colors": torch.Tensor (N, 3) -- RGB in [0, 1], or None +# "confidence": torch.Tensor (N,) -- raw confidence per point, or None + + +def _da3_unproject(depth: torch.Tensor, K: torch.Tensor) -> torch.Tensor: + """Pixel-space K⁻¹ unprojection: (H,W) depth → (H,W,3) point map in OpenCV space.""" + H, W = depth.shape + u = torch.arange(W, dtype=torch.float32, device=depth.device) + v = torch.arange(H, dtype=torch.float32, device=depth.device) + u, v = torch.meshgrid(u, v, indexing='xy') # both (H, W) + pix = torch.stack([u, v, torch.ones_like(u)], dim=-1) # (H, W, 3) + rays = torch.einsum('ij,hwj->hwi', torch.linalg.inv(K.to(depth.device)), pix) + return rays * depth.unsqueeze(-1) # (H, W, 3) + + +def _da3_default_K(H: int, W: int) -> torch.Tensor: + """Fallback ~60° FOV pinhole K for mono-mode DA3 (no intrinsics in geometry).""" + fx = fy = float(W) * 0.7 + return torch.tensor([[fx, 0.0, (W - 1) / 2.0], + [0.0, fy, (H - 1) / 2.0], + [0.0, 0.0, 1.0]], dtype=torch.float32) + + +def _da3_get_K(geometry: dict, b: int, H: int, W: int) -> torch.Tensor: + """Return pixel-space K for batch element b, falling back to a default estimate.""" + if "intrinsics" in geometry: + # shape (1, S, 3, 3) - leading scene dimension from the multiview head + return geometry["intrinsics"][0, b].float() + logging.getLogger("comfy").warning( + "DA3_GEOMETRY has no intrinsics (mono-mode model). " + "Using a ~60° FOV estimate; 3-D reconstruction may be inaccurate." + ) + return _da3_default_K(H, W) + + +def _da3_get_extrinsic(geometry: dict, b: int) -> torch.Tensor | None: + """Return the world-to-camera extrinsic for batch element b, or None in mono mode. + + The model outputs (1, S, 3, 4) [R|t] matrices; the fallback identity is (4, 4). + _da3_apply_extrinsic handles both shapes via [:3, :3] / [:3, 3] slicing. + """ + if "extrinsics" not in geometry: + return None + return geometry["extrinsics"][0, b].float() + + +def _da3_apply_extrinsic(points_cam: torch.Tensor, E: torch.Tensor) -> torch.Tensor: + """Transform (H,W,3) OpenCV camera-space points to world space.""" + E = E.to(points_cam.device).float() + if not torch.isfinite(E).all(): + logging.getLogger("comfy").warning( + "DA3 extrinsic matrix contains non-finite values (pose estimation may have failed). " + "Falling back to camera-space coordinates." + ) + return points_cam + H, W, _ = points_cam.shape + R = E[:3, :3] # (3, 3) rotation + t = E[:3, 3] # (3,) translation + R_inv = R.T # rotation inverse = transpose for orthogonal R + t_inv = -(R_inv @ t) # (3,) + pts = points_cam.reshape(-1, 3) # (N, 3) + pts_world = pts @ R_inv.T + t_inv # (N, 3) + return pts_world.reshape(H, W, 3) + + +def _normalize_confidence(conf: torch.Tensor) -> torch.Tensor: + """Map raw confidence to [0, 1] per image.""" + B = conf.shape[0] + out = [] + for i in range(B): + c = conf[i] + c_min, c_max = c.min(), c.max() + out.append((c - c_min) / (c_max - c_min) if c_max > c_min else torch.ones_like(c)) + return torch.stack(out, dim=0) + + +def _da3_build_mask(geometry: dict, b: int, H: int, W: int, confidence_threshold: float, use_sky_mask: bool) -> torch.Tensor: + """Build (H,W) bool keep-mask from sky probability and confidence.""" + mask = torch.ones(H, W, dtype=torch.bool) + if use_sky_mask and "sky" in geometry: + mask = mask & (geometry["sky"][b] < 0.5) + if "confidence" in geometry and confidence_threshold > 0.0: + conf_norm = _normalize_confidence(geometry["confidence"][b:b + 1])[0] + mask = mask & (conf_norm >= confidence_threshold) + return mask + + +class LoadDA3Model(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadDA3Model", + display_name="Load Depth Anything 3", + category="model/loaders", + inputs=[ + io.Combo.Input( + "model_name", + options=folder_paths.get_filename_list("geometry_estimation"), + ), + io.Combo.Input( + "weight_dtype", + options=["default", "fp16", "bf16", "fp32"], + default="default", + ), + ], + outputs=[DA3ModelType.Output()], + ) + + @classmethod + def execute(cls, model_name, weight_dtype) -> io.NodeOutput: + model_options = {} + if weight_dtype == "fp16": + model_options["dtype"] = torch.float16 + elif weight_dtype == "bf16": + model_options["dtype"] = torch.bfloat16 + elif weight_dtype == "fp32": + model_options["dtype"] = torch.float32 + + path = folder_paths.get_full_path_or_raise("geometry_estimation", model_name) + model = comfy.sd.load_diffusion_model(path, model_options=model_options) + return io.NodeOutput(model) + + +def _run_da3(model_patcher, image: torch.Tensor, process_res: int, method: str = "upper_bound_resize"): + """Run DA3 on (B,H,W,3), returns depth/conf/sky at original resolution (or None).""" + assert image.ndim == 4 and image.shape[-1] == 3, f"expected (B,H,W,3) IMAGE; got {tuple(image.shape)}" + + B, H, W, _ = image.shape + mm.load_model_gpu(model_patcher) + diffusion = model_patcher.model.diffusion_model + device = mm.get_torch_device() + dtype = diffusion.dtype if diffusion.dtype is not None else torch.float32 + + depths, confs, skies = [], [], [] + for i in range(B): + single = image[i:i + 1].to(device) + x = da3_preprocess.preprocess_image(single, process_res=process_res, method=method) + x = x.to(dtype=dtype) + with torch.no_grad(): + out = diffusion(x) + + depth_lr = out["depth"] + depth_full = torch.nn.functional.interpolate( + depth_lr.unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + depths.append(depth_full) + + if "depth_conf" in out: + conf_full = torch.nn.functional.interpolate( + out["depth_conf"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + confs.append(conf_full) + if "sky" in out: + sky_full = torch.nn.functional.interpolate( + out["sky"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + skies.append(sky_full) + + depth = torch.cat(depths, dim=0) + confidence = torch.cat(confs, dim=0) if confs else None + sky = torch.cat(skies, dim=0) if skies else None + return depth, confidence, sky + + +class DA3Inference(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3Inference", + search_aliases=["depth", "geometry", "da3", "depth anything", "monocular", "pointmap", "sky", "3d", "metric depth", "disparity"], + display_name="Run Depth Anything 3", + category="image/geometry estimation", + description="Run Depth Anything 3 on an image. In multi-view mode each image is treated as a separate view of the same scene.", + inputs=[ + DA3ModelType.Input("da3_model"), + io.Image.Input("image"), + io.Int.Input("resolution", default=504, min=140, max=2520, step=14, + tooltip="Resolution the model runs at (longest side, multiple of 14).\n" + "Lower = faster / less VRAM.\n" + "Higher = more detail.\n" + "Output is upsampled back to the original size."), + io.Combo.Input("resize_method", options=["upper_bound_resize", "lower_bound_resize"], default="upper_bound_resize", + tooltip="upper_bound_resize: scale so the longest side = resolution (caps memory, default).\n" + "lower_bound_resize: scale so the shortest side = resolution (preserves more detail on tall/wide images, uses more memory)."), + io.DynamicCombo.Input("mode", tooltip="mono: single view image (works with any model variant).\n" + "multiview: all images processed together for geometric consistency + camera pose (for Small/Base models only).", + options=[ + io.DynamicCombo.Option("mono", []), + io.DynamicCombo.Option("multiview", [ + io.Combo.Input("ref_view_strategy", options=["saddle_balanced", "saddle_sim_range", "first", "middle"], default="saddle_balanced", + tooltip="Which view acts as the geometric anchor.\n" + "- saddle_balanced: the view most 'average' across all others (best general choice).\n" + "- saddle_sim_range: the view most visually distinct from the others.\n" + "- first / middle: fixed positional picks."), + io.Combo.Input("pose_method", options=["cam_dec", "ray_pose"], default="cam_dec", + tooltip="How the camera field-of-view is estimated (for Small/Base models only).\n" + "- cam_dec: learned from image features.\n" + "- ray_pose: derived geometrically from the model's 3D ray output.\n" + "Affects perspective correctness of the 3D output. Try both if results look distorted."), + ]), + ]), + ], + outputs=[ + DA3Geometry.Output("da3_geometry", tooltip="Dictionary of non-normalized tensors.\n" + "Always has the keys: depth, image, mode.\n" + "Optional keys: sky (for Mono/Metric), confidence (for Small/Base), extrinsics + intrinsics (for multi-view)."), + ], + ) + + @classmethod + def execute(cls, da3_model, image, resolution, resize_method, mode) -> io.NodeOutput: + mode_val = mode["mode"] # "mono" or "multiview" + + if mode_val == "mono": + return cls._execute_mono(da3_model, image, resolution, resize_method) + + # Capability checks for multi-view mode. + diffusion = da3_model.model.diffusion_model + pose_method = mode["pose_method"] + ref_view_strategy = mode["ref_view_strategy"] + + has_cam_dec = diffusion.cam_dec is not None + has_dualdpt = diffusion.head_type == "dualdpt" + + if not has_cam_dec and not has_dualdpt: + raise ValueError( + "multi-view mode requires Small or Base model. The loaded model " + f"(head_type='{diffusion.head_type}') does not support cross-view " + "attention or camera pose estimation. Switch mode to 'mono', or " + "load Small or Base model for mult-view." + ) + + if pose_method == "cam_dec" and not has_cam_dec: + raise ValueError( + "pose_method='cam_dec' requires a camera decoder, but the loaded " + f"model (head_type='{diffusion.head_type}') does not have one. " + "Use pose_method='ray_pose' instead." + ) + if pose_method == "ray_pose" and not has_dualdpt: + raise ValueError( + "pose_method='ray_pose' requires a DualDPT head, but the loaded " + f"model has a '{diffusion.head_type}' head. " + "Use pose_method='cam_dec' instead." + ) + + return cls._execute_multiview( + da3_model, image, resolution, resize_method, + ref_view_strategy, pose_method, + ) + + @classmethod + def _execute_mono(cls, model, image, resolution, resize_method) -> io.NodeOutput: + depth, confidence, sky = _run_da3(model, image, resolution, method=resize_method) + + geometry: dict = { + "depth": depth.contiguous(), + "image": image[..., :3].cpu(), + "mode": "mono", + } + if sky is not None: + geometry["sky"] = sky.contiguous() + if confidence is not None: + geometry["confidence"] = confidence.contiguous() + return io.NodeOutput(geometry) + + @classmethod + def _execute_multiview(cls, model, image, resolution, resize_method, ref_view_strategy, pose_method) -> io.NodeOutput: + assert image.ndim == 4 and image.shape[-1] == 3, \ + f"expected (B,H,W,3) IMAGE; got {tuple(image.shape)}" + S, H, W, _ = image.shape + + mm.load_model_gpu(model) + diffusion = model.model.diffusion_model + device = mm.get_torch_device() + dtype = diffusion.dtype if diffusion.dtype is not None else torch.float32 + + # All views in a single forward pass: (1, S, 3, H', W'). + x = image.to(device) + x = da3_preprocess.preprocess_image(x, process_res=resolution, method=resize_method) + x = x.to(dtype=dtype).unsqueeze(0) + + use_ray_pose = (pose_method == "ray_pose") + with torch.no_grad(): + out = diffusion(x, use_ray_pose=use_ray_pose, ref_view_strategy=ref_view_strategy) + + depth = torch.nn.functional.interpolate( + out["depth"].float().unsqueeze(1), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + + sky = None + if "sky" in out: + sky = torch.nn.functional.interpolate( + out["sky"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + + if "extrinsics" in out and "intrinsics" in out: + extrinsics = out["extrinsics"].float().cpu() + intrinsics = out["intrinsics"].float().cpu() + else: + extrinsics = torch.eye(4)[None, None].expand(1, S, 4, 4).clone() + intrinsics = torch.eye(3)[None, None].expand(1, S, 3, 3).clone() + + geometry: dict = { + "depth": depth.contiguous(), + "image": image[..., :3].cpu(), + "mode": "multiview", + "extrinsics": extrinsics.contiguous(), + "intrinsics": intrinsics.contiguous(), + } + if sky is not None: + geometry["sky"] = sky.contiguous() + if "depth_conf" in out: + conf = torch.nn.functional.interpolate( + out["depth_conf"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + geometry["confidence"] = conf.contiguous() + return io.NodeOutput(geometry) + + +class DA3Render(io.ComfyNode): + """Render a visualization from a DA3_GEOMETRY packet.""" + + _DEPTH_RENDER_INPUTS = [ + io.Combo.Input("normalization", + options=["v2_style", "min_max", "raw"], + default="v2_style", + tooltip="- v2_style: mean/std normalisation for perceptually balanced results (default).\n" + "- min_max: stretches the full depth range to [0, 1] for maximum contrast.\n" + "- raw: no scaling,preserves metric units for Metric model."), + io.Boolean.Input("apply_sky_clip", default=False, + tooltip="Clip sky-region depth to the 99th percentile of foreground depth before normalisation. " + "Requires a sky key in the da3_geometry input (for Mono/Metric models only)."), + ] + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3Render", + display_name="Render Depth Anything 3", + category="image/geometry estimation", + description="Render a depth map, confidence map, or sky mask from Depth Anything 3 geometry data.", + inputs=[ + DA3Geometry.Input("da3_geometry"), + io.DynamicCombo.Input("output", + tooltip="- depth: normalised greyscale depth image.\n" + "- depth_colored: depth mapped through the Turbo colormap.\n" + "- sky_mask: sky probability in [0, 1] (for Mono/Metric models only).\n" + "- confidence: normalised depth confidence (for Small/Base models only).", + options=[ + io.DynamicCombo.Option("depth", cls._DEPTH_RENDER_INPUTS), + io.DynamicCombo.Option("depth_colored", cls._DEPTH_RENDER_INPUTS), + io.DynamicCombo.Option("sky_mask", [ + io.Boolean.Input("colored", default=False, tooltip="Apply the Turbo colormap to the sky mask."), + ]), + io.DynamicCombo.Option("confidence", [ + io.Boolean.Input("colored", default=False, tooltip="Apply the Turbo colormap to the confidence map."), + ]), + ]), + ], + outputs=[io.Image.Output()], + ) + + @classmethod + def execute(cls, da3_geometry, output) -> io.NodeOutput: + output_val = output["output"] + + if output_val in ("depth", "depth_colored"): + normalization = output["normalization"] + apply_sky_clip = output["apply_sky_clip"] + if apply_sky_clip and "sky" not in da3_geometry: + raise ValueError( + "apply_sky_clip=True requires a sky tensor in the da3_geometry input, but none is present. " + "Run with Mono/Metric models or set apply_sky_clip=False." + ) + depth = da3_geometry["depth"] + sky = da3_geometry.get("sky") + if apply_sky_clip and sky is not None: + depth = torch.stack([ + da3_preprocess.apply_sky_aware_clip(depth[i], sky[i]) + for i in range(depth.shape[0]) + ], dim=0) + grey = cls._depth_to_image(depth, sky, normalization) # (B,H,W,3) greyscale + result = _turbo(grey[..., 0]) if output_val == "depth_colored" else grey + + elif output_val == "sky_mask": + if "sky" not in da3_geometry: + raise ValueError("geometry has no sky output; run with Mono/Metric models.") + sky = da3_geometry["sky"] + if output["colored"]: + result = _turbo(sky) + else: + result = sky.unsqueeze(-1).expand(*sky.shape, 3).contiguous() + + elif output_val == "confidence": + if "confidence" not in da3_geometry: + raise ValueError("da3_geometry has no confidence output; run with Small/Base models.") + conf = _normalize_confidence(da3_geometry["confidence"]) + if output["colored"]: + result = _turbo(conf) + else: + result = conf.unsqueeze(-1).expand(*conf.shape, 3).contiguous() + + else: + raise ValueError(f"Unknown output mode: {output_val}") + + return io.NodeOutput(result.float()) + + @staticmethod + def _depth_to_image(depth: torch.Tensor, sky_for_norm: torch.Tensor | None, normalization: str) -> torch.Tensor: + """Normalise depth and pack as an (B,H,W,3) image tensor.""" + + N = depth.shape[0] + if normalization == "v2_style": + norm = torch.stack([ + da3_preprocess.normalize_depth_v2_style( + depth[i], sky_for_norm[i] if sky_for_norm is not None else None) + for i in range(N) + ], dim=0) + elif normalization == "min_max": + norm = da3_preprocess.normalize_depth_min_max(depth) + else: + norm = depth + + out = norm.unsqueeze(-1).repeat(1, 1, 1, 3) + if normalization != "raw": + out = out.clamp(0.0, 1.0) + return out.contiguous() + + +class DA3GeometryToMesh(io.ComfyNode): + """Convert a DA3_GEOMETRY packet into a Types.MESH by unprojecting depth and triangulating.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3GeometryToMesh", + search_aliases=["da3", "depth anything", "mesh", "geometry", "3d", "triangulate"], + display_name="Convert DA3 Geometry to Mesh", + category="image/geometry estimation", + description="Convert a depth map into a triangulated 3D mesh.", + inputs=[ + DA3Geometry.Input("da3_geometry"), + io.Int.Input("batch_index", default=0, min=0, max=4096, tooltip="Which image of a batch to convert. Per-image vertex counts differ so batches cannot be stacked."), + io.Int.Input("decimation", default=1, min=1, max=8, tooltip="Vertex stride. 1 = full resolution, 2 = half, etc."), + io.Float.Input("discontinuity_threshold", default=0.04, min=0.0, max=1.0, step=0.01, tooltip="Drop triangles whose 3x3 depth span exceeds this fraction. 0 = off."), + io.Float.Input("confidence_threshold", default=0.1, min=0.0, max=1.0, step=0.01, + tooltip="Exclude pixels whose per-image normalised confidence is below this value (0 = keep all, 1 = keep only the single most confident pixel). " + "Used when the geometry has a confidence map (Small/Base models)."), + io.Boolean.Input("use_sky_mask", default=True, tooltip="Exclude sky-probability pixels (sky >= 0.5) from the mesh. Used when the geometry has a sky map (Mono/Metric models)."), + io.Boolean.Input("texture", default=True, tooltip="Use the source image as a base color texture."), + ], + outputs=[io.Mesh.Output()], + ) + + @classmethod + def execute(cls, da3_geometry, batch_index, decimation, discontinuity_threshold, confidence_threshold, use_sky_mask, texture) -> io.NodeOutput: + depth_all = da3_geometry["depth"] # (B, H, W) + B = depth_all.shape[0] + if batch_index >= B: + raise ValueError(f"batch_index {batch_index} is out of range; DA3_GEOMETRY has batch size {B}.") + + depth = depth_all[batch_index] # (H, W) + H, W = depth.shape + + # NaN/inf depth would propagate silently through unproject and produce an + # empty mesh; replace them with 0 here so those pixels are later excluded + # by the isfinite check inside triangulate_grid_mesh. + depth = depth.clone() + n_bad = (~torch.isfinite(depth)).sum().item() + if n_bad: + logging.getLogger("comfy").warning( + f"DA3GeometryToMesh: depth[{batch_index}] has {n_bad} non-finite pixels " + f"({100*n_bad/(H*W):.1f}%) - zeroed before unproject." + ) + depth[~torch.isfinite(depth)] = 0.0 + logging.getLogger("comfy").debug( + f"DA3GeometryToMesh: depth[{batch_index}] range " + f"[{depth.min():.4g}, {depth.max():.4g}], mean={depth.mean():.4g}" + ) + + K = _da3_get_K(da3_geometry, batch_index, H, W) + points = _da3_unproject(depth, K) # (H, W, 3) in OpenCV camera space + + # Apply world-to-camera inverse so multi-view frames share a common world frame. + E = _da3_get_extrinsic(da3_geometry, batch_index) + if E is not None: + points = _da3_apply_extrinsic(points, E) + + # Mask invalid pixels by setting them to inf so triangulate_grid_mesh skips them. + mask = _da3_build_mask(da3_geometry, batch_index, H, W, confidence_threshold, use_sky_mask) + # Also exclude pixels where depth was invalid. + mask = mask & (depth_all[batch_index] > 0) & torch.isfinite(depth_all[batch_index]) + points = points.clone() + points[~mask] = float('inf') + + verts, faces, uvs = triangulate_grid_mesh( + points, + decimation=decimation, + discontinuity_threshold=discontinuity_threshold, + depth=depth, + ) + if verts.shape[0] == 0 or faces.shape[0] == 0: + raise ValueError( + "DA3GeometryToMesh produced an empty mesh. " + "Try raising discontinuity_threshold, lowering confidence_threshold, " + "or disabling use_sky_mask." + ) + + # OpenCV (X right, Y down, Z forward) → glTF (X right, Y up, Z back). + # Same transform as MoGePointMapToMesh perspective branch. + verts = verts * torch.tensor([1.0, -1.0, -1.0], dtype=verts.dtype) + faces = faces[:, [0, 2, 1]].contiguous() + + tex = da3_geometry["image"][batch_index:batch_index + 1] if texture else None + mesh = Types.MESH( + vertices=verts.unsqueeze(0), + faces=faces.unsqueeze(0), + uvs=uvs.unsqueeze(0), + texture=tex, + ) + return io.NodeOutput(mesh) + + +class DA3GeometryToPointCloud(io.ComfyNode): + """Unproject a DA3_GEOMETRY depth map into a filtered DA3_POINT_CLOUD.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3GeometryToPointCloud", + search_aliases=["da3", "depth anything", "point cloud", "pointcloud", "3d", "geometry"], + display_name="Convert DA3 Geometry to Point Cloud", + category="image/geometry estimation", + description="Convert a depth map into a 3D point cloud.", + inputs=[ + DA3Geometry.Input("da3_geometry"), + io.Int.Input("batch_index", default=0, min=0, max=4096, tooltip="Which image of a batch to convert."), + io.Float.Input("confidence_threshold", default=0.1, min=0.0, max=1.0, step=0.01, + tooltip="Exclude pixels whose per-image normalised confidence is below this value (0 = keep all). Used when the geometry has a confidence map (Small/Base models)."), + io.Boolean.Input("use_sky_mask", default=True, + tooltip="Exclude sky-probability pixels (sky >= 0.5). Used when the geometry has a sky map (Mono/Metric models)."), + io.Int.Input("downsample", default=1, min=1, max=16, + tooltip="Take every Nth pixel (1 = full resolution). Higher values give fewer points and faster processing."), + ], + # TODO: add a proper PointCloud output type + outputs=[DA3PointCloud.Output(display_name="point_cloud")], + ) + + @classmethod + def execute(cls, da3_geometry, batch_index, confidence_threshold, use_sky_mask, downsample) -> io.NodeOutput: + depth_all = da3_geometry["depth"] # (B, H, W) + B = depth_all.shape[0] + if batch_index >= B: + raise ValueError(f"batch_index {batch_index} is out of range; DA3_GEOMETRY has batch size {B}.") + + depth = depth_all[batch_index].clone() # (H, W) + depth[~torch.isfinite(depth)] = 0.0 + H, W = depth.shape + + K = _da3_get_K(da3_geometry, batch_index, H, W) + + if downsample > 1: + depth = depth[::downsample, ::downsample].contiguous() + # Scale intrinsics to the downsampled grid. + K = K.clone() + K[0, :] /= downsample + K[1, :] /= downsample + + H_ds, W_ds = depth.shape + points = _da3_unproject(depth, K) # (H_ds, W_ds, 3) in OpenCV camera space + + # Apply world-to-camera inverse so multi-view frames share a common world frame. + E = _da3_get_extrinsic(da3_geometry, batch_index) + if E is not None: + points = _da3_apply_extrinsic(points, E) + + # Rebuild mask at downsampled resolution. + mask = _da3_build_mask(da3_geometry, batch_index, H, W, confidence_threshold, use_sky_mask) + if downsample > 1: + mask = mask[::downsample, ::downsample] + + mask = mask & torch.isfinite(depth) + + # OpenCV → glTF: flip Y and Z. + points_gltf = points.clone() + points_gltf[..., 1] *= -1.0 + points_gltf[..., 2] *= -1.0 + + pts_flat = points_gltf.reshape(-1, 3)[mask.reshape(-1)] + + colors_flat = None + if "image" in da3_geometry: + img = da3_geometry["image"][batch_index] # (H, W, 3) + if downsample > 1: + img = img[::downsample, ::downsample] + colors_flat = img.reshape(-1, 3)[mask.reshape(-1)] + + conf_flat = None + if "confidence" in da3_geometry: + conf = da3_geometry["confidence"][batch_index] # (H, W) + if downsample > 1: + conf = conf[::downsample, ::downsample] + conf_flat = conf.reshape(-1)[mask.reshape(-1)] + + if pts_flat.shape[0] == 0: + raise ValueError( + "DA3GeometryToPointCloud produced zero points after filtering. " + "Try lowering confidence_threshold or disabling use_sky_mask." + ) + + return io.NodeOutput({ + "points": pts_flat, + "colors": colors_flat, + "confidence": conf_flat, + }) + + +class DA3Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LoadDA3Model, + DA3Inference, + DA3Render, + DA3GeometryToMesh, + # DA3GeometryToPointCloud, # Keep this commented out for now until we have a proper PointCloud output type + ] + + +async def comfy_entrypoint() -> DA3Extension: + return DA3Extension() diff --git a/comfy_extras/nodes_easycache.py b/comfy_extras/nodes_easycache.py index 923c2bb05..9e907d371 100644 --- a/comfy_extras/nodes_easycache.py +++ b/comfy_extras/nodes_easycache.py @@ -363,7 +363,7 @@ class EasyCacheNode(io.ComfyNode): node_id="EasyCache", display_name="EasyCache", description="Native EasyCache implementation.", - category="advanced/debug/model", + category="advanced/debug", is_experimental=True, inputs=[ io.Model.Input("model", tooltip="The model to add EasyCache to."), @@ -496,7 +496,7 @@ class LazyCacheNode(io.ComfyNode): node_id="LazyCache", display_name="LazyCache", description="A homebrew version of EasyCache - even 'easier' version of EasyCache to implement. Overall works worse than EasyCache, but better in some rare cases AND universal compatibility with everything in ComfyUI.", - category="advanced/debug/model", + category="advanced/debug", is_experimental=True, inputs=[ io.Model.Input("model", tooltip="The model to add LazyCache to."), diff --git a/comfy_extras/nodes_edit_model.py b/comfy_extras/nodes_edit_model.py index 36da66f34..d0d20ae7a 100644 --- a/comfy_extras/nodes_edit_model.py +++ b/comfy_extras/nodes_edit_model.py @@ -8,7 +8,8 @@ class ReferenceLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ReferenceLatent", - category="advanced/conditioning/edit_models", + display_name="Set Reference Latent", + category="model/conditioning", description="This node sets the guiding latent for an edit model. If the model supports it you can chain multiple to set multiple reference images.", inputs=[ io.Conditioning.Input("conditioning"), diff --git a/comfy_extras/nodes_flux.py b/comfy_extras/nodes_flux.py index afc663b22..e9986c9e7 100644 --- a/comfy_extras/nodes_flux.py +++ b/comfy_extras/nodes_flux.py @@ -13,7 +13,7 @@ class CLIPTextEncodeFlux(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeFlux", - category="advanced/conditioning/flux", + category="model/conditioning/flux", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), @@ -40,7 +40,7 @@ class EmptyFlux2LatentImage(io.ComfyNode): return io.Schema( node_id="EmptyFlux2LatentImage", display_name="Empty Flux 2 Latent", - category="model/latent", + category="model/latent/flux", inputs=[ io.Int.Input("width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -61,7 +61,7 @@ class FluxGuidance(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FluxGuidance", - category="advanced/conditioning/flux", + category="model/conditioning/flux", inputs=[ io.Conditioning.Input("conditioning"), io.Float.Input("guidance", default=3.5, min=0.0, max=100.0, step=0.1), @@ -84,7 +84,7 @@ class FluxDisableGuidance(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FluxDisableGuidance", - category="advanced/conditioning/flux", + category="model/conditioning/flux", description="This node completely disables the guidance embed on Flux and Flux like models", inputs=[ io.Conditioning.Input("conditioning"), @@ -128,7 +128,7 @@ class FluxKontextImageScale(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FluxKontextImageScale", - category="advanced/conditioning/flux", + category="model/conditioning/flux", description="This node resizes the image to one that is more optimal for flux kontext.", inputs=[ io.Image.Input("image"), @@ -156,7 +156,7 @@ class FluxKontextMultiReferenceLatentMethod(io.ComfyNode): return io.Schema( node_id="FluxKontextMultiReferenceLatentMethod", display_name="Edit Model Reference Method", - category="advanced/conditioning/flux", + category="model/conditioning/flux", inputs=[ io.Conditioning.Input("conditioning"), io.Combo.Input( @@ -245,6 +245,11 @@ class KV_Attn_Input: cache_key = "{}_{}".format(extra_options["block_type"], extra_options["block_index"]) if cache_key in self.cache: kk, vv = self.cache[cache_key] + + # Fix batch size changing. + kk = comfy.utils.repeat_to_batch_size(kk, k.shape[0]) + vv = comfy.utils.repeat_to_batch_size(vv, v.shape[0]) + self.set_cache = False return {"q": q, "k": torch.cat((k, kk), dim=2), "v": torch.cat((v, vv), dim=2)} diff --git a/comfy_extras/nodes_frame_interpolation.py b/comfy_extras/nodes_frame_interpolation.py index 4d5bca17e..44708e5ec 100644 --- a/comfy_extras/nodes_frame_interpolation.py +++ b/comfy_extras/nodes_frame_interpolation.py @@ -77,7 +77,7 @@ class FrameInterpolate(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FrameInterpolate", - display_name="Frame Interpolate", + display_name="Run Frame Interpolation Model", category="video", search_aliases=["rife", "film", "frame interpolation", "slow motion", "interpolate frames", "vfi"], inputs=[ diff --git a/comfy_extras/nodes_gaussian_splat.py b/comfy_extras/nodes_gaussian_splat.py index 2ba3a3820..116c14fde 100644 --- a/comfy_extras/nodes_gaussian_splat.py +++ b/comfy_extras/nodes_gaussian_splat.py @@ -488,7 +488,7 @@ class SplatToFile3D(IO.ComfyNode): "spz: Niantic gzip-compressed (~10x smaller), base color only " ), ], - outputs=[IO.File3DAny.Output(display_name="model_3d")], + outputs=[IO.File3DSplatAny.Output(display_name="model_3d")], ) @classmethod @@ -516,7 +516,7 @@ class File3DToSplat(IO.ComfyNode): inputs=[ IO.MultiType.Input( IO.File3DAny.Input("model_3d"), - types=[IO.File3DPLY, IO.File3DSPLAT, IO.File3DKSPLAT, IO.File3DSPZ], + types=[IO.File3DSplatAny, IO.File3DPLY, IO.File3DSPLAT, IO.File3DKSPLAT, IO.File3DSPZ], tooltip="A gaussian splat 3D file", ), ], diff --git a/comfy_extras/nodes_glsl.py b/comfy_extras/nodes_glsl.py index ea7420a73..c7161973a 100644 --- a/comfy_extras/nodes_glsl.py +++ b/comfy_extras/nodes_glsl.py @@ -1,85 +1,68 @@ import os import sys import re +import ctypes import logging -import ctypes.util -import importlib.util from typing import TypedDict import numpy as np import torch import nodes +import comfy_angle from comfy_api.latest import ComfyExtension, io, ui from typing_extensions import override -from utils.install_util import get_missing_requirements_message logger = logging.getLogger(__name__) -def _check_opengl_availability(): - """Early check for OpenGL availability. Raises RuntimeError if unlikely to work.""" - logger.debug("_check_opengl_availability: starting") - missing = [] +def _preload_angle(): + egl_path = comfy_angle.get_egl_path() + gles_path = comfy_angle.get_glesv2_path() - # Check Python packages (using find_spec to avoid importing) - logger.debug("_check_opengl_availability: checking for glfw package") - if importlib.util.find_spec("glfw") is None: - missing.append("glfw") + if sys.platform == "win32": + angle_dir = comfy_angle.get_lib_dir() + os.add_dll_directory(angle_dir) + os.environ["PATH"] = angle_dir + os.pathsep + os.environ.get("PATH", "") - logger.debug("_check_opengl_availability: checking for OpenGL package") - if importlib.util.find_spec("OpenGL") is None: - missing.append("PyOpenGL") - - if missing: - raise RuntimeError( - f"OpenGL dependencies not available.\n{get_missing_requirements_message()}\n" - ) - - # On Linux without display, check if headless backends are available - logger.debug(f"_check_opengl_availability: platform={sys.platform}") - if sys.platform.startswith("linux"): - has_display = os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY") - logger.debug(f"_check_opengl_availability: has_display={bool(has_display)}") - if not has_display: - # Check for EGL or OSMesa libraries - logger.debug("_check_opengl_availability: checking for EGL library") - has_egl = ctypes.util.find_library("EGL") - logger.debug("_check_opengl_availability: checking for OSMesa library") - has_osmesa = ctypes.util.find_library("OSMesa") - - # Error disabled for CI as it fails this check - # if not has_egl and not has_osmesa: - # raise RuntimeError( - # "GLSL Shader node: No display and no headless backend (EGL/OSMesa) found.\n" - # "See error below for installation instructions." - # ) - logger.debug(f"Headless mode: EGL={'yes' if has_egl else 'no'}, OSMesa={'yes' if has_osmesa else 'no'}") - - logger.debug("_check_opengl_availability: completed") + mode = 0 if sys.platform == "win32" else ctypes.RTLD_GLOBAL + ctypes.CDLL(str(egl_path), mode=mode) + ctypes.CDLL(str(gles_path), mode=mode) -# Run early check at import time -logger.debug("nodes_glsl: running _check_opengl_availability at import time") -_check_opengl_availability() - -# OpenGL modules - initialized lazily when context is created -gl = None -glfw = None -EGL = None +# Pre-load ANGLE *before* any PyOpenGL import so that the EGL platform +# plugin picks up ANGLE's libEGL / libGLESv2 instead of system libs. +_preload_angle() +os.environ.setdefault("PYOPENGL_PLATFORM", "egl") -def _import_opengl(): - """Import OpenGL module. Called after context is created.""" - global gl - if gl is None: - logger.debug("_import_opengl: importing OpenGL.GL") - import OpenGL.GL as _gl - gl = _gl - logger.debug("_import_opengl: import completed") - return gl +import OpenGL +OpenGL.USE_ACCELERATE = False +def _patch_find_library(): + """PyOpenGL's EGL platform looks for 'EGL' and 'GLESv2' by short name + via ctypes.util.find_library, but ANGLE ships as 'libEGL' and + 'libGLESv2'. Patch find_library to return the full ANGLE paths so + PyOpenGL loads the same libraries we pre-loaded.""" + if sys.platform == "linux": + return + import ctypes.util + _orig = ctypes.util.find_library + def _patched(name): + if name == 'EGL': + return comfy_angle.get_egl_path() + if name == 'GLESv2': + return comfy_angle.get_glesv2_path() + return _orig(name) + ctypes.util.find_library = _patched + + +_patch_find_library() + +from OpenGL import EGL +from OpenGL import GLES3 as gl + class SizeModeInput(TypedDict): size_mode: str width: int @@ -102,7 +85,7 @@ MAX_OUTPUTS = 4 # fragColor0-3 (MRT) # (-1,-1)---(3,-1) # # v_texCoord is computed from clip space: * 0.5 + 0.5 maps (-1,1) -> (0,1) -VERTEX_SHADER = """#version 330 core +VERTEX_SHADER = """#version 300 es out vec2 v_texCoord; void main() { vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3)); @@ -126,14 +109,99 @@ void main() { """ -def _convert_es_to_desktop(source: str) -> str: - """Convert GLSL ES (WebGL) shader source to desktop GLSL 330 core.""" - # Remove any existing #version directive - source = re.sub(r"#version\s+\d+(\s+es)?\s*\n?", "", source, flags=re.IGNORECASE) - # Remove precision qualifiers (not needed in desktop GLSL) - source = re.sub(r"precision\s+(lowp|mediump|highp)\s+\w+\s*;\s*\n?", "", source) - # Prepend desktop GLSL version - return "#version 330 core\n" + source + +def _egl_attribs(*values): + """Build an EGL_NONE-terminated EGLint attribute array.""" + vals = list(values) + [EGL.EGL_NONE] + return (ctypes.c_int32 * len(vals))(*vals) + + +# EGL platform extension constants +EGL_PLATFORM_ANGLE_ANGLE = 0x3202 +EGL_PLATFORM_ANGLE_TYPE_ANGLE = 0x3203 +EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE = 0x3450 +EGL_MESA_PLATFORM_SURFACELESS = 0x31DD + + +_eglGetPlatformDisplayEXT = None + +def _get_egl_platform_display_ext(platform, native_display, attribs): + """Call eglGetPlatformDisplayEXT via ctypes (extension, not in PyOpenGL).""" + global _eglGetPlatformDisplayEXT + if _eglGetPlatformDisplayEXT is None: + from OpenGL import platform as _plat + egl_lib = _plat.PLATFORM.EGL + _get_proc = egl_lib.eglGetProcAddress + _get_proc.restype = ctypes.c_void_p + _get_proc.argtypes = [ctypes.c_char_p] + ptr = _get_proc(b"eglGetPlatformDisplayEXT") + if not ptr: + return None + func_type = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p) + _eglGetPlatformDisplayEXT = func_type(ptr) + + raw = _eglGetPlatformDisplayEXT(platform, native_display, attribs) + if not raw: + return None + return ctypes.cast(raw, EGL.EGLDisplay) + + +def _get_egl_display(): + """Get an EGL display, trying the default first then ANGLE's Vulkan + platform for headless environments without a display server.""" + failures = [] + + # Try the default display first (works when X11/Wayland is available) + display = EGL.eglGetDisplay(EGL.EGL_DEFAULT_DISPLAY) + if display: + major, minor = ctypes.c_int32(0), ctypes.c_int32(0) + try: + if EGL.eglInitialize(display, ctypes.byref(major), ctypes.byref(minor)): + return display, major.value, minor.value + except Exception as e: + failures.append(f"default: {e}") + + logger.info("Default EGL display unavailable, trying headless fallbacks") + + # Headless fallback strategies, tried in order: + headless_strategies = [ + ("surfaceless", EGL_MESA_PLATFORM_SURFACELESS, None, None), + ("ANGLE Vulkan", EGL_PLATFORM_ANGLE_ANGLE, None, + _egl_attribs(EGL_PLATFORM_ANGLE_TYPE_ANGLE, EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE)), + ] + + for name, platform, native_display, attribs in headless_strategies: + display = _get_egl_platform_display_ext(platform, native_display, attribs) + if not display: + failures.append(f"{name}: eglGetPlatformDisplayEXT returned no display") + continue + major, minor = ctypes.c_int32(0), ctypes.c_int32(0) + try: + if EGL.eglInitialize(display, ctypes.byref(major), ctypes.byref(minor)): + logger.info(f"Using EGL {name} platform (headless)") + return display, major.value, minor.value + failures.append(f"{name}: eglInitialize returned false") + except Exception as e: + failures.append(f"{name}: {e}") + continue + + details = "\n".join(f" - {f}" for f in failures) + raise RuntimeError( + "Failed to initialize EGL display.\n" + "No display server and no headless EGL platform available.\n" + f"Tried:\n{details}\n" + "Ensure GPU drivers are installed or set DISPLAY for a virtual framebuffer." + ) + + +def _gl_str(name): + """Get an OpenGL string parameter.""" + v = gl.glGetString(name) + if not v: + return "Unknown" + if isinstance(v, bytes): + return v.decode(errors="replace") + return ctypes.string_at(v).decode(errors="replace") def _detect_output_count(source: str) -> int: @@ -159,163 +227,8 @@ def _detect_pass_count(source: str) -> int: return 1 -def _init_glfw(): - """Initialize GLFW. Returns (window, glfw_module). Raises RuntimeError on failure.""" - logger.debug("_init_glfw: starting") - # On macOS, glfw.init() must be called from main thread or it hangs forever - if sys.platform == "darwin": - logger.debug("_init_glfw: skipping on macOS") - raise RuntimeError("GLFW backend not supported on macOS") - - logger.debug("_init_glfw: importing glfw module") - import glfw as _glfw - - logger.debug("_init_glfw: calling glfw.init()") - if not _glfw.init(): - raise RuntimeError("glfw.init() failed") - - try: - logger.debug("_init_glfw: setting window hints") - _glfw.window_hint(_glfw.VISIBLE, _glfw.FALSE) - _glfw.window_hint(_glfw.CONTEXT_VERSION_MAJOR, 3) - _glfw.window_hint(_glfw.CONTEXT_VERSION_MINOR, 3) - _glfw.window_hint(_glfw.OPENGL_PROFILE, _glfw.OPENGL_CORE_PROFILE) - - logger.debug("_init_glfw: calling create_window()") - window = _glfw.create_window(64, 64, "ComfyUI GLSL", None, None) - if not window: - raise RuntimeError("glfw.create_window() failed") - - logger.debug("_init_glfw: calling make_context_current()") - _glfw.make_context_current(window) - logger.debug("_init_glfw: completed successfully") - return window, _glfw - except Exception: - logger.debug("_init_glfw: failed, terminating glfw") - _glfw.terminate() - raise - - -def _init_egl(): - """Initialize EGL for headless rendering. Returns (display, context, surface, EGL_module). Raises RuntimeError on failure.""" - logger.debug("_init_egl: starting") - from OpenGL import EGL as _EGL - from OpenGL.EGL import ( - eglGetDisplay, eglInitialize, eglChooseConfig, eglCreateContext, - eglMakeCurrent, eglCreatePbufferSurface, eglBindAPI, - eglTerminate, eglDestroyContext, eglDestroySurface, - EGL_DEFAULT_DISPLAY, EGL_NO_CONTEXT, EGL_NONE, - EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, - EGL_RED_SIZE, EGL_GREEN_SIZE, EGL_BLUE_SIZE, EGL_ALPHA_SIZE, EGL_DEPTH_SIZE, - EGL_WIDTH, EGL_HEIGHT, EGL_OPENGL_API, - ) - logger.debug("_init_egl: imports completed") - - display = None - context = None - surface = None - - try: - logger.debug("_init_egl: calling eglGetDisplay()") - display = eglGetDisplay(EGL_DEFAULT_DISPLAY) - if display == _EGL.EGL_NO_DISPLAY: - raise RuntimeError("eglGetDisplay() failed") - - logger.debug("_init_egl: calling eglInitialize()") - major, minor = _EGL.EGLint(), _EGL.EGLint() - if not eglInitialize(display, major, minor): - display = None # Not initialized, don't terminate - raise RuntimeError("eglInitialize() failed") - logger.debug(f"_init_egl: EGL version {major.value}.{minor.value}") - - config_attribs = [ - EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, - EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, - EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, - EGL_DEPTH_SIZE, 0, EGL_NONE - ] - configs = (_EGL.EGLConfig * 1)() - num_configs = _EGL.EGLint() - if not eglChooseConfig(display, config_attribs, configs, 1, num_configs) or num_configs.value == 0: - raise RuntimeError("eglChooseConfig() failed") - config = configs[0] - logger.debug(f"_init_egl: config chosen, num_configs={num_configs.value}") - - if not eglBindAPI(EGL_OPENGL_API): - raise RuntimeError("eglBindAPI() failed") - - logger.debug("_init_egl: calling eglCreateContext()") - context_attribs = [ - _EGL.EGL_CONTEXT_MAJOR_VERSION, 3, - _EGL.EGL_CONTEXT_MINOR_VERSION, 3, - _EGL.EGL_CONTEXT_OPENGL_PROFILE_MASK, _EGL.EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, - EGL_NONE - ] - context = eglCreateContext(display, config, EGL_NO_CONTEXT, context_attribs) - if context == EGL_NO_CONTEXT: - raise RuntimeError("eglCreateContext() failed") - - logger.debug("_init_egl: calling eglCreatePbufferSurface()") - pbuffer_attribs = [EGL_WIDTH, 64, EGL_HEIGHT, 64, EGL_NONE] - surface = eglCreatePbufferSurface(display, config, pbuffer_attribs) - if surface == _EGL.EGL_NO_SURFACE: - raise RuntimeError("eglCreatePbufferSurface() failed") - - logger.debug("_init_egl: calling eglMakeCurrent()") - if not eglMakeCurrent(display, surface, surface, context): - raise RuntimeError("eglMakeCurrent() failed") - - logger.debug("_init_egl: completed successfully") - return display, context, surface, _EGL - - except Exception: - logger.debug("_init_egl: failed, cleaning up") - # Clean up any resources on failure - if surface is not None: - eglDestroySurface(display, surface) - if context is not None: - eglDestroyContext(display, context) - if display is not None: - eglTerminate(display) - raise - - -def _init_osmesa(): - """Initialize OSMesa for software rendering. Returns (context, buffer). Raises RuntimeError on failure.""" - import ctypes - - logger.debug("_init_osmesa: starting") - os.environ["PYOPENGL_PLATFORM"] = "osmesa" - - logger.debug("_init_osmesa: importing OpenGL.osmesa") - from OpenGL import GL as _gl - from OpenGL.osmesa import ( - OSMesaCreateContextExt, OSMesaMakeCurrent, OSMesaDestroyContext, - OSMESA_RGBA, - ) - logger.debug("_init_osmesa: imports completed") - - ctx = OSMesaCreateContextExt(OSMESA_RGBA, 24, 0, 0, None) - if not ctx: - raise RuntimeError("OSMesaCreateContextExt() failed") - - width, height = 64, 64 - buffer = (ctypes.c_ubyte * (width * height * 4))() - - logger.debug("_init_osmesa: calling OSMesaMakeCurrent()") - if not OSMesaMakeCurrent(ctx, buffer, _gl.GL_UNSIGNED_BYTE, width, height): - OSMesaDestroyContext(ctx) - raise RuntimeError("OSMesaMakeCurrent() failed") - - logger.debug("_init_osmesa: completed successfully") - return ctx, buffer - - class GLContext: - """Manages OpenGL context and resources for shader execution. - - Tries backends in order: GLFW (desktop) → EGL (headless GPU) → OSMesa (software). - """ + """Manages an OpenGL ES 3.0 context via EGL/ANGLE (singleton).""" _instance = None _initialized = False @@ -327,131 +240,105 @@ class GLContext: def __init__(self): if GLContext._initialized: - logger.debug("GLContext.__init__: already initialized, skipping") return - logger.debug("GLContext.__init__: starting initialization") - - global glfw, EGL - import time start = time.perf_counter() - self._backend = None - self._window = None - self._egl_display = None - self._egl_context = None - self._egl_surface = None - self._osmesa_ctx = None - self._osmesa_buffer = None + self._display = None + self._surface = None + self._context = None self._vao = None - # Try backends in order: GLFW → EGL → OSMesa - errors = [] - - logger.debug("GLContext.__init__: trying GLFW backend") try: - self._window, glfw = _init_glfw() - self._backend = "glfw" - logger.debug("GLContext.__init__: GLFW backend succeeded") - except Exception as e: - logger.debug(f"GLContext.__init__: GLFW backend failed: {e}") - errors.append(("GLFW", e)) + self._display, self._egl_major, self._egl_minor = _get_egl_display() - if self._backend is None: - logger.debug("GLContext.__init__: trying EGL backend") - try: - self._egl_display, self._egl_context, self._egl_surface, EGL = _init_egl() - self._backend = "egl" - logger.debug("GLContext.__init__: EGL backend succeeded") - except Exception as e: - logger.debug(f"GLContext.__init__: EGL backend failed: {e}") - errors.append(("EGL", e)) + if not EGL.eglBindAPI(EGL.EGL_OPENGL_ES_API): + raise RuntimeError("eglBindAPI(EGL_OPENGL_ES_API) failed") - if self._backend is None: - logger.debug("GLContext.__init__: trying OSMesa backend") - try: - self._osmesa_ctx, self._osmesa_buffer = _init_osmesa() - self._backend = "osmesa" - logger.debug("GLContext.__init__: OSMesa backend succeeded") - except Exception as e: - logger.debug(f"GLContext.__init__: OSMesa backend failed: {e}") - errors.append(("OSMesa", e)) + config = EGL.EGLConfig() + n_configs = ctypes.c_int32(0) + if not EGL.eglChooseConfig( + self._display, + _egl_attribs( + EGL.EGL_RENDERABLE_TYPE, EGL.EGL_OPENGL_ES3_BIT, + EGL.EGL_SURFACE_TYPE, EGL.EGL_PBUFFER_BIT, + EGL.EGL_RED_SIZE, 8, EGL.EGL_GREEN_SIZE, 8, + EGL.EGL_BLUE_SIZE, 8, EGL.EGL_ALPHA_SIZE, 8, + ), + ctypes.byref(config), 1, ctypes.byref(n_configs), + ) or n_configs.value == 0: + raise RuntimeError("eglChooseConfig() failed") - if self._backend is None: - if sys.platform == "win32": - platform_help = ( - "Windows: Ensure GPU drivers are installed and display is available.\n" - " CPU-only/headless mode is not supported on Windows." - ) - elif sys.platform == "darwin": - platform_help = ( - "macOS: GLFW is not supported.\n" - " Install OSMesa via Homebrew: brew install mesa\n" - " Then: pip install PyOpenGL PyOpenGL-accelerate" - ) - else: - platform_help = ( - "Linux: Install one of these backends:\n" - " Desktop: sudo apt install libgl1-mesa-glx libglfw3\n" - " Headless with GPU: sudo apt install libegl1-mesa libgl1-mesa-dri\n" - " Headless (CPU): sudo apt install libosmesa6" - ) - - error_details = "\n".join(f" {name}: {err}" for name, err in errors) - raise RuntimeError( - f"Failed to create OpenGL context.\n\n" - f"Backend errors:\n{error_details}\n\n" - f"{platform_help}" + self._surface = EGL.eglCreatePbufferSurface( + self._display, config, + _egl_attribs(EGL.EGL_WIDTH, 64, EGL.EGL_HEIGHT, 64), ) + if not self._surface: + raise RuntimeError("eglCreatePbufferSurface() failed") - # Now import OpenGL.GL (after context is current) - logger.debug("GLContext.__init__: importing OpenGL.GL") - _import_opengl() + self._context = EGL.eglCreateContext( + self._display, config, EGL.EGL_NO_CONTEXT, + _egl_attribs(EGL.EGL_CONTEXT_CLIENT_VERSION, 3), + ) + if not self._context: + raise RuntimeError("eglCreateContext() failed") - # Create VAO (required for core profile, but OSMesa may use compat profile) - logger.debug("GLContext.__init__: creating VAO") - try: - vao = gl.glGenVertexArrays(1) - gl.glBindVertexArray(vao) - self._vao = vao # Only store after successful bind - logger.debug("GLContext.__init__: VAO created successfully") - except Exception as e: - logger.debug(f"GLContext.__init__: VAO creation failed (may be expected for OSMesa): {e}") - # OSMesa with older Mesa may not support VAOs - # Clean up if we created but couldn't bind - if vao: - try: - gl.glDeleteVertexArrays(1, [vao]) - except Exception: - pass + if not EGL.eglMakeCurrent(self._display, self._surface, self._surface, self._context): + raise RuntimeError("eglMakeCurrent() failed") + + self._vao = gl.glGenVertexArrays(1) + gl.glBindVertexArray(self._vao) + + except Exception: + self._cleanup() + raise elapsed = (time.perf_counter() - start) * 1000 - # Log device info - renderer = gl.glGetString(gl.GL_RENDERER) - vendor = gl.glGetString(gl.GL_VENDOR) - version = gl.glGetString(gl.GL_VERSION) - renderer = renderer.decode() if renderer else "Unknown" - vendor = vendor.decode() if vendor else "Unknown" - version = version.decode() if version else "Unknown" + renderer = _gl_str(gl.GL_RENDERER) + vendor = _gl_str(gl.GL_VENDOR) + version = _gl_str(gl.GL_VERSION) GLContext._initialized = True - logger.info(f"GLSL context initialized in {elapsed:.1f}ms ({self._backend}) - {renderer} ({vendor}), GL {version}") + logger.info(f"GLSL context initialized in {elapsed:.1f}ms - EGL {self._egl_major}.{self._egl_minor}, {renderer} ({vendor}), GL {version}") def make_current(self): - if self._backend == "glfw": - glfw.make_context_current(self._window) - elif self._backend == "egl": - from OpenGL.EGL import eglMakeCurrent - eglMakeCurrent(self._egl_display, self._egl_surface, self._egl_surface, self._egl_context) - elif self._backend == "osmesa": - from OpenGL.osmesa import OSMesaMakeCurrent - OSMesaMakeCurrent(self._osmesa_ctx, self._osmesa_buffer, gl.GL_UNSIGNED_BYTE, 64, 64) - + if not EGL.eglMakeCurrent(self._display, self._surface, self._surface, self._context): + err = EGL.eglGetError() + raise RuntimeError(f"eglMakeCurrent() failed (EGL error: 0x{err:04X})") if self._vao is not None: gl.glBindVertexArray(self._vao) + def _cleanup(self): + if not self._display: + return + try: + if self._vao is not None: + gl.glDeleteVertexArrays(1, [self._vao]) + self._vao = None + except Exception: + pass + try: + EGL.eglMakeCurrent(self._display, EGL.EGL_NO_SURFACE, EGL.EGL_NO_SURFACE, EGL.EGL_NO_CONTEXT) + except Exception: + pass + try: + if self._context: + EGL.eglDestroyContext(self._display, self._context) + except Exception: + pass + try: + if self._surface: + EGL.eglDestroySurface(self._display, self._surface) + except Exception: + pass + try: + EGL.eglTerminate(self._display) + except Exception: + pass + self._display = None + def _compile_shader(source: str, shader_type: int) -> int: """Compile a shader and return its ID.""" @@ -459,8 +346,10 @@ def _compile_shader(source: str, shader_type: int) -> int: gl.glShaderSource(shader, source) gl.glCompileShader(shader) - if gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS) != gl.GL_TRUE: - error = gl.glGetShaderInfoLog(shader).decode() + if not gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS): + error = gl.glGetShaderInfoLog(shader) + if isinstance(error, bytes): + error = error.decode(errors="replace") gl.glDeleteShader(shader) raise RuntimeError(f"Shader compilation failed:\n{error}") @@ -484,8 +373,10 @@ def _create_program(vertex_source: str, fragment_source: str) -> int: gl.glDeleteShader(vertex_shader) gl.glDeleteShader(fragment_shader) - if gl.glGetProgramiv(program, gl.GL_LINK_STATUS) != gl.GL_TRUE: - error = gl.glGetProgramInfoLog(program).decode() + if not gl.glGetProgramiv(program, gl.GL_LINK_STATUS): + error = gl.glGetProgramInfoLog(program) + if isinstance(error, bytes): + error = error.decode(errors="replace") gl.glDeleteProgram(program) raise RuntimeError(f"Program linking failed:\n{error}") @@ -530,9 +421,6 @@ def _render_shader_batch( ctx = GLContext() ctx.make_current() - # Convert from GLSL ES to desktop GLSL 330 - fragment_source = _convert_es_to_desktop(fragment_code) - # Detect how many outputs the shader actually uses num_outputs = _detect_output_count(fragment_code) @@ -558,9 +446,9 @@ def _render_shader_batch( try: # Compile shaders (once for all batches) try: - program = _create_program(VERTEX_SHADER, fragment_source) + program = _create_program(VERTEX_SHADER, fragment_code) except RuntimeError: - logger.error(f"Fragment shader:\n{fragment_source}") + logger.error(f"Fragment shader:\n{fragment_code}") raise gl.glUseProgram(program) @@ -723,13 +611,13 @@ def _render_shader_batch( gl.glDrawArrays(gl.GL_TRIANGLES, 0, 3) # Read back outputs for this batch - # (glGetTexImage is synchronous, implicitly waits for rendering) + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo) batch_outputs = [] - for tex in output_textures: - gl.glBindTexture(gl.GL_TEXTURE_2D, tex) - data = gl.glGetTexImage(gl.GL_TEXTURE_2D, 0, gl.GL_RGBA, gl.GL_FLOAT) - img = np.frombuffer(data, dtype=np.float32).reshape(height, width, 4) - batch_outputs.append(img[::-1, :, :].copy()) + for i in range(num_outputs): + gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + i) + buf = np.empty((height, width, 4), dtype=np.float32) + gl.glReadPixels(0, 0, width, height, gl.GL_RGBA, gl.GL_FLOAT, buf) + batch_outputs.append(buf[::-1, :, :].copy()) # Pad with black images for unused outputs black_img = np.zeros((height, width, 4), dtype=np.float32) @@ -750,18 +638,18 @@ def _render_shader_batch( gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0) gl.glUseProgram(0) - for tex in input_textures: - gl.glDeleteTextures(int(tex)) - for tex in curve_textures: - gl.glDeleteTextures(int(tex)) - for tex in output_textures: - gl.glDeleteTextures(int(tex)) - for tex in ping_pong_textures: - gl.glDeleteTextures(int(tex)) + if input_textures: + gl.glDeleteTextures(len(input_textures), input_textures) + if curve_textures: + gl.glDeleteTextures(len(curve_textures), curve_textures) + if output_textures: + gl.glDeleteTextures(len(output_textures), output_textures) + if ping_pong_textures: + gl.glDeleteTextures(len(ping_pong_textures), ping_pong_textures) if fbo is not None: gl.glDeleteFramebuffers(1, [fbo]) - for pp_fbo in ping_pong_fbos: - gl.glDeleteFramebuffers(1, [pp_fbo]) + if ping_pong_fbos: + gl.glDeleteFramebuffers(len(ping_pong_fbos), ping_pong_fbos) if program is not None: gl.glDeleteProgram(program) diff --git a/comfy_extras/nodes_hidream.py b/comfy_extras/nodes_hidream.py index e345fe51d..65248561b 100644 --- a/comfy_extras/nodes_hidream.py +++ b/comfy_extras/nodes_hidream.py @@ -11,8 +11,9 @@ class QuadrupleCLIPLoader(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="QuadrupleCLIPLoader", - category="advanced/loaders", - description="[Recipes]\n\nhidream: long clip-l, long clip-g, t5xxl, llama_8b_3.1_instruct", + display_name="Load CLIP (Quadruple)", + category="model/loaders", + description="Recipes:\nhidream: long clip-l, long clip-g, t5xxl, llama_8b_3.1_instruct", inputs=[ io.Combo.Input("clip_name1", options=folder_paths.get_filename_list("text_encoders")), io.Combo.Input("clip_name2", options=folder_paths.get_filename_list("text_encoders")), @@ -38,8 +39,9 @@ class CLIPTextEncodeHiDream(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeHiDream", + display_name="CLIP Text Encode (HiDream)", search_aliases=["hidream prompt"], - category="advanced/conditioning", + category="model/conditioning/hidream", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_hidream_o1.py b/comfy_extras/nodes_hidream_o1.py index 8648d2e26..85693fce6 100644 --- a/comfy_extras/nodes_hidream_o1.py +++ b/comfy_extras/nodes_hidream_o1.py @@ -14,7 +14,7 @@ class EmptyHiDreamO1LatentImage(io.ComfyNode): return io.Schema( node_id="EmptyHiDreamO1LatentImage", display_name="Empty HiDream-O1 Latent Image", - category="model/latent/image", + category="model/latent/hidream", description=( "Empty pixel-space latent for HiDream-O1-Image. The model was " "trained at ~4 megapixels; lower resolutions go off-distribution " @@ -47,7 +47,7 @@ class HiDreamO1ReferenceImages(io.ComfyNode): return io.Schema( node_id="HiDreamO1ReferenceImages", display_name="HiDream-O1 Reference Images", - category="model/conditioning/image", + category="model/conditioning/hidream", description=( "Attach 1-10 reference images to conditioning, one for edit instruction" "or multiple for subject-driven personalization." @@ -117,7 +117,7 @@ class HiDreamO1PatchSeamSmoothing(io.ComfyNode): return io.Schema( node_id="HiDreamO1PatchSeamSmoothing", display_name="HiDream-O1 Patch Seam Smoothing", - category="advanced/model", + category="model/patch/hidream", is_experimental=True, description=( "Average the model output across multiple shifted patch-grid " diff --git a/comfy_extras/nodes_hunyuan.py b/comfy_extras/nodes_hunyuan.py index 16fff12af..8df2c8908 100644 --- a/comfy_extras/nodes_hunyuan.py +++ b/comfy_extras/nodes_hunyuan.py @@ -14,7 +14,8 @@ class CLIPTextEncodeHunyuanDiT(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeHunyuanDiT", - category="advanced/conditioning", + display_name="CLIP Text Encode (Hunyuan Image)", + category="model/conditioning/hunyuan image", inputs=[ io.Clip.Input("clip"), io.String.Input("bert", multiline=True, dynamic_prompts=True), @@ -41,7 +42,7 @@ class EmptyHunyuanLatentVideo(io.ComfyNode): return io.Schema( node_id="EmptyHunyuanLatentVideo", display_name="Empty HunyuanVideo 1.0 Latent", - category="model/latent/video", + category="model/latent/hunyuan video", inputs=[ io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -67,6 +68,7 @@ class EmptyHunyuanVideo15Latent(EmptyHunyuanLatentVideo): schema = super().define_schema() schema.node_id = "EmptyHunyuanVideo15Latent" schema.display_name = "Empty HunyuanVideo 1.5 Latent" + schema.category = "model/latent/hunyuan video" return schema @classmethod @@ -81,7 +83,7 @@ class HunyuanVideo15ImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="HunyuanVideo15ImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -132,7 +134,7 @@ class HunyuanVideo15SuperResolution(io.ComfyNode): return io.Schema( node_id="HunyuanVideo15SuperResolution", display_name="Hunyuan Video 1.5 Super Resolution", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -227,7 +229,7 @@ class HunyuanVideo15LatentUpscaleWithModel(io.ComfyNode): return io.Schema( node_id="HunyuanVideo15LatentUpscaleWithModel", display_name="Hunyuan Video 15 Latent Upscale With Model", - category="model/latent", + category="model/latent/hunyhuan video", inputs=[ io.LatentUpscaleModel.Input("model"), io.Latent.Input("samples"), @@ -276,7 +278,7 @@ class TextEncodeHunyuanVideo_ImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeHunyuanVideo_ImageToVideo", - category="advanced/conditioning", + category="model/conditioning/hunyuan video", inputs=[ io.Clip.Input("clip"), io.ClipVisionOutput.Input("clip_vision_output"), @@ -308,7 +310,7 @@ class HunyuanImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="HunyuanImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Vae.Input("vae"), @@ -359,7 +361,7 @@ class EmptyHunyuanImageLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyHunyuanImageLatent", - category="model/latent", + category="model/latent/hunyuan image", inputs=[ io.Int.Input("width", default=2048, min=64, max=nodes.MAX_RESOLUTION, step=32), io.Int.Input("height", default=2048, min=64, max=nodes.MAX_RESOLUTION, step=32), @@ -384,7 +386,7 @@ class HunyuanRefinerLatent(io.ComfyNode): return io.Schema( node_id="HunyuanRefinerLatent", display_name="Hunyuan Latent Refiner", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), diff --git a/comfy_extras/nodes_hunyuan3d.py b/comfy_extras/nodes_hunyuan3d.py index 60e530626..c5fa946cc 100644 --- a/comfy_extras/nodes_hunyuan3d.py +++ b/comfy_extras/nodes_hunyuan3d.py @@ -12,7 +12,7 @@ class EmptyLatentHunyuan3Dv2(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="EmptyLatentHunyuan3Dv2", - category="model/latent/3d", + category="model/latent/hunyuan 3d", inputs=[ IO.Int.Input("resolution", default=3072, min=1, max=8192), IO.Int.Input("batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch."), @@ -35,7 +35,7 @@ class Hunyuan3Dv2Conditioning(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="Hunyuan3Dv2Conditioning", - category="model/conditioning/3d_models", + category="model/conditioning/hunyuan 3d", inputs=[ IO.ClipVisionOutput.Input("clip_vision_output"), ], @@ -60,7 +60,7 @@ class Hunyuan3Dv2ConditioningMultiView(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="Hunyuan3Dv2ConditioningMultiView", - category="model/conditioning/3d_models", + category="model/conditioning/hunyuan 3d", inputs=[ IO.ClipVisionOutput.Input("front", optional=True), IO.ClipVisionOutput.Input("left", optional=True), @@ -97,7 +97,7 @@ class VAEDecodeHunyuan3D(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="VAEDecodeHunyuan3D", - category="model/latent/3d", + category="model/latent/hunyuan 3d", inputs=[ IO.Latent.Input("samples"), IO.Vae.Input("vae"), diff --git a/comfy_extras/nodes_ideogram4.py b/comfy_extras/nodes_ideogram4.py new file mode 100644 index 000000000..4070db17c --- /dev/null +++ b/comfy_extras/nodes_ideogram4.py @@ -0,0 +1,64 @@ +"""Ideogram 4 sampling helper +""" + +import math + +import torch +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + +_LOGSNR_MIN = -15.0 +_LOGSNR_MAX = 18.0 + + +def _logit_normal_schedule(u, mean, std): + # Reference time (0=noise..1=clean) via the probit/ndtri quantile. + u = torch.as_tensor(u, dtype=torch.float64) + t = 1.0 - torch.special.expit(mean + std * torch.special.ndtri(u)) + t_min = 1.0 / (1.0 + math.exp(0.5 * _LOGSNR_MAX)) + t_max = 1.0 / (1.0 + math.exp(0.5 * _LOGSNR_MIN)) + return t.clamp(t_min, t_max) + + +def ideogram4_sigmas(num_steps, width, height, mu, std): + """Descending sigmas (len num_steps+1) for the reference schedule. + + mu + the resolution term form the logSNR shift; std is the spread. + """ + mean = mu + 0.5 * math.log((width * height) / (512 * 512)) + u = torch.linspace(0.0, 1.0, num_steps + 1, dtype=torch.float64) + sigmas = (1.0 - _logit_normal_schedule(u, mean, std)).flip(0) + sigmas[-1] = 0.0 # clamp leaves ~6e-4; force full denoise + return sigmas.to(torch.float32) + + +class Ideogram4Scheduler(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="Ideogram4Scheduler", + display_name="Ideogram 4 Scheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=200), + io.Int.Input("width", default=1024, min=256, max=8192, step=16), + io.Int.Input("height", default=1024, min=256, max=8192, step=16), + io.Float.Input("mu", default=0.0, min=-10.0, max=10.0, step=0.05), + io.Float.Input("std", default=1.75, min=0.1, max=5.0, step=0.05), + ], + outputs=[io.Sigmas.Output()], + ) + + @classmethod + def execute(cls, steps, width, height, mu, std) -> io.NodeOutput: + return io.NodeOutput(ideogram4_sigmas(steps, width, height, mu, std)) + + +class Ideogram4Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [Ideogram4Scheduler] + + +async def comfy_entrypoint() -> Ideogram4Extension: + return Ideogram4Extension() diff --git a/comfy_extras/nodes_images.py b/comfy_extras/nodes_images.py index 469a7be55..7011d9c13 100644 --- a/comfy_extras/nodes_images.py +++ b/comfy_extras/nodes_images.py @@ -214,11 +214,13 @@ class SaveAnimatedWEBP(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] ) @classmethod def execute(cls, images, fps, filename_prefix, lossless, quality, method, num_frames=0) -> IO.NodeOutput: return IO.NodeOutput( + images, ui=UI.ImageSaveHelper.get_save_animated_webp_ui( images=images, filename_prefix=filename_prefix, @@ -230,8 +232,6 @@ class SaveAnimatedWEBP(IO.ComfyNode): ) ) - save_images = execute # TODO: remove - class SaveAnimatedPNG(IO.ComfyNode): @@ -249,11 +249,13 @@ class SaveAnimatedPNG(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] ) @classmethod def execute(cls, images, fps, compress_level, filename_prefix="ComfyUI") -> IO.NodeOutput: return IO.NodeOutput( + images, ui=UI.ImageSaveHelper.get_save_animated_png_ui( images=images, filename_prefix=filename_prefix, @@ -263,8 +265,6 @@ class SaveAnimatedPNG(IO.ComfyNode): ) ) - save_images = execute # TODO: remove - class ImageStitch(IO.ComfyNode): """Upstreamed from https://github.com/kijai/ComfyUI-KJNodes""" @@ -513,6 +513,7 @@ class SaveSVGNode(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.SVG.Output("svg")], ) @classmethod @@ -562,9 +563,7 @@ class SaveSVGNode(IO.ComfyNode): results.append(UI.SavedResult(filename=file, subfolder=subfolder, type=IO.FolderType.output)) counter += 1 - return IO.NodeOutput(ui={"images": results}) - - save_svg = execute # TODO: remove + return IO.NodeOutput(svg, ui={"images": results}) class GetImageSize(IO.ComfyNode): @@ -845,15 +844,18 @@ class ImageMergeTileList(IO.ComfyNode): # Format specifications # --------------------------------------------------------------------------- -# Maps (file_format, bit_depth, has_alpha) -> (numpy dtype scale, av pixel format, -# stream pix_fmt). Keeps the encode path declarative instead of branchy. +# Maps (file_format, bit_depth, num_channels) -> (quantization scale, numpy dtype, +# av frame pix_fmt, stream pix_fmt). Keeps the encode path declarative instead of branchy. _FORMAT_SPECS = { - ("png", "8-bit", False): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "rgb24", "stream_fmt": "rgb24"}, - ("png", "8-bit", True): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "rgba", "stream_fmt": "rgba"}, - ("png", "16-bit", False): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "rgb48le", "stream_fmt": "rgb48be"}, - ("png", "16-bit", True): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "rgba64le", "stream_fmt": "rgba64be"}, - ("exr", "32-bit float", False): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "gbrpf32le", "stream_fmt": "gbrpf32le"}, - ("exr", "32-bit float", True): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "gbrapf32le", "stream_fmt": "gbrapf32le"}, + ("png", "8-bit", 1): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "gray", "stream_fmt": "gray"}, + ("png", "8-bit", 3): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "rgb24", "stream_fmt": "rgb24"}, + ("png", "8-bit", 4): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "rgba", "stream_fmt": "rgba"}, + ("png", "16-bit", 1): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "gray16le", "stream_fmt": "gray16be"}, + ("png", "16-bit", 3): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "rgb48le", "stream_fmt": "rgb48be"}, + ("png", "16-bit", 4): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "rgba64le", "stream_fmt": "rgba64be"}, + ("exr", "32-bit float", 1): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "grayf32le", "stream_fmt": "grayf32le"}, + ("exr", "32-bit float", 3): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "gbrpf32le", "stream_fmt": "gbrpf32le"}, + ("exr", "32-bit float", 4): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "gbrapf32le", "stream_fmt": "gbrapf32le"}, } @@ -892,10 +894,11 @@ def hlg_to_linear(t: torch.Tensor) -> torch.Tensor: return torch.cat([hlg_to_linear(rgb), alpha], dim=-1) # Piecewise: sqrt branch below 0.5, log branch above. - # Clamp inside the log branch so negative / out-of-range values don't blow up; + # Clamp the log branch at the 0.5 branch point (not above it) so the + # unselected lane stays finite in exp() without altering selected values; # values above 1.0 are allowed and extrapolate naturally. low = (t ** 2) / 3.0 - high = (torch.exp((t.clamp(min=_HLG_C) - _HLG_C) / _HLG_A) + _HLG_B) / 12.0 + high = (torch.exp((t.clamp(min=0.5) - _HLG_C) / _HLG_A) + _HLG_B) / 12.0 return torch.where(t <= 0.5, low, high) @@ -1088,7 +1091,8 @@ def _encode_image( bit_depth: str, colorspace: str, ) -> bytes: - """Encode a single HxWxC tensor to PNG or EXR bytes in memory. + """Encode a single HxWxC (or channel-less HxW grayscale) tensor to PNG or + EXR bytes in memory. Grayscale is written as single-channel PNG / Y-only EXR. For EXR the input is interpreted according to `colorspace` and converted to scene-linear (EXR's convention) before writing: @@ -1102,10 +1106,16 @@ def _encode_image( For PNG, colorspace selection does not modify pixels — PNG is delivered sRGB-encoded and there is no PNG path for wide-gamut HDR in this node. """ + if img_tensor.ndim == 2: + img_tensor = img_tensor.unsqueeze(-1) # Some nodes emit grayscale as (H, W) with no channel dim, mask-style. height, width, num_channels = img_tensor.shape - has_alpha = num_channels == 4 - spec = _FORMAT_SPECS[(file_format, bit_depth, has_alpha)] + spec = _FORMAT_SPECS.get((file_format, bit_depth, num_channels)) + if spec is None: + raise ValueError( + f"No {file_format}/{bit_depth} encoder for {num_channels}-channel images: " + "supported channel counts are 1 (grayscale), 3 (RGB) and 4 (RGBA)." + ) if spec["dtype"] == np.float32: # EXR path: preserve full range, no clamp. @@ -1157,40 +1167,27 @@ class SaveImageAdvanced(IO.ComfyNode): IO.String.Input( "filename_prefix", default="ComfyUI", - tooltip=( - "The prefix for the file to save. May include formatting tokens " - "such as %date:yyyy-MM-dd% or %Empty Latent Image.width%." - ), + tooltip=("The prefix for the file to save. May include formatting tokens such as %date:yyyy-MM-dd% or %Empty Latent Image.width%."), ), IO.DynamicCombo.Input( "format", options=[ IO.DynamicCombo.Option("png", [ - IO.Combo.Input("bit_depth", options=["8-bit", "16-bit"], - default="8-bit", advanced=True), - IO.Combo.Input("input_color_space", options=["sRGB"], - default="sRGB", advanced=True), + IO.Combo.Input("bit_depth", options=["8-bit", "16-bit"], default="8-bit", advanced=True), + IO.Combo.Input("input_color_space", options=["sRGB"], default="sRGB", advanced=True), ]), IO.DynamicCombo.Option("exr", [ - IO.Combo.Input("bit_depth", options=["32-bit float"], - default="32-bit float", advanced=True), + IO.Combo.Input("bit_depth", options=["32-bit float"], default="32-bit float", advanced=True), IO.Combo.Input( "input_color_space", options=["sRGB", "HDR", "linear"], default="sRGB", advanced=True, tooltip=( - "Colorspace of the input tensor. The EXR is " - "always written as scene-linear in the matching " - "gamut.\n" - " 'sRGB' — input is sRGB-encoded Rec.709; " - "the inverse sRGB EOTF is applied.\n" - " 'HDR' — input is HLG-encoded Rec.2020 " - "(BT.2100); the inverse HLG OETF is applied " - "to get scene-linear light.\n" - " 'linear' — input is already scene-linear " - "(Rec.709 primaries); written through unchanged. " - "Use this for renderer/compositor output." + "Colorspace of the input tensor. The EXR is always written as scene-linear in the matching gamut.\n" + "sRGB — input is sRGB-encoded Rec.709; the inverse sRGB EOTF is applied.\n" + "HDR — input is HLG-encoded Rec.2020 (BT.2100); the inverse HLG OETF is applied to get scene-linear light.\n" + "linear — input is already scene-linear (Rec.709 primaries); written through unchanged. Use this for renderer/compositor output." ), ), ]), @@ -1200,6 +1197,7 @@ class SaveImageAdvanced(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] ) @classmethod @@ -1237,7 +1235,7 @@ class SaveImageAdvanced(IO.ComfyNode): results.append({"filename": file, "subfolder": subfolder, "type": "output"}) counter += 1 - return IO.NodeOutput(ui={"images": results}) + return IO.NodeOutput(images, ui={"images": results}) class ImagesExtension(ComfyExtension): diff --git a/comfy_extras/nodes_json_prompt.py b/comfy_extras/nodes_json_prompt.py new file mode 100644 index 000000000..206f5aa71 --- /dev/null +++ b/comfy_extras/nodes_json_prompt.py @@ -0,0 +1,77 @@ +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import normalize_palette + + +class BuildJsonPromptIdeogram(io.ComfyNode): + @classmethod + def define_schema(cls): + color_palette = io.Colors.Input( + "color_palette", + socketless=False, + tooltip="Hex color codes that steer the image's dominant colors. Up to 16 entries.", + ) + return io.Schema( + node_id="BuildJsonPromptIdeogram", + display_name="Build JSON Prompt (Ideogram)", + category="text", + description="Build a JSON prompt for the Ideogram 4 model.", + inputs=[ + io.Array.Input("element", tooltip="Prompt elements from the node Create Bounding Boxes."), + io.String.Input("high_level_description", multiline=True, default="", + tooltip="Optional description of the image in one or two sentences. Strongly recommended."), + io.String.Input("background", multiline=True, default="", + tooltip="Mandatory description of the image background or environment."), + io.DynamicCombo.Input("style", options=[ + io.DynamicCombo.Option("none", []), + io.DynamicCombo.Option("photo", [io.String.Input("photo", default="", tooltip="Camera or lens details for photographic outputs (e.g. 35mm, f/1.4, bokeh).")]), + io.DynamicCombo.Option("art_style", [io.String.Input("art_style", default="", tooltip="Art style description (e.g. flat vector illustration, bold outlines).")]), + ]), + io.String.Input("aesthetics", default="", tooltip="Mandatory aesthetic keywords (e.g. moody, cinematic, desaturated)."), + io.String.Input("lighting", default="", tooltip="Mandatory lighting description (e.g. golden hour, rim light, dramatic shadows)."), + io.String.Input("medium", default="", tooltip="Mandatory medium type (e.g. photograph, illustration, 3d_render, painting, graphic_design). When style = photo, set to photograph."), + color_palette, + ], + outputs=[io.Dict.Output(display_name="prompt")], + is_experimental=True, + ) + + @classmethod + def execute(cls, element, style, high_level_description="", background="", + aesthetics="", lighting="", medium="", color_palette=None) -> io.NodeOutput: + elements = element if isinstance(element, list) else [] + kind = style.get("style", "none") if isinstance(style, dict) else "none" + photo = style.get("photo", "") if isinstance(style, dict) else "" + art_style = style.get("art_style", "") if isinstance(style, dict) else "" + palette = normalize_palette(color_palette or []) + + caption: dict = {} + if high_level_description.strip(): + caption["high_level_description"] = high_level_description + if kind != "none": + style_desc: dict = {"aesthetics": aesthetics, "lighting": lighting} + if kind == "photo": + style_desc["photo"] = photo + style_desc["medium"] = medium + else: + style_desc["medium"] = medium + style_desc["art_style"] = art_style + if palette: + style_desc["color_palette"] = palette + caption["style_description"] = style_desc + caption["compositional_deconstruction"] = { + "background": background, + "elements": elements, + } + return io.NodeOutput(caption) + + +class JsonPromptExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [BuildJsonPromptIdeogram] + + +async def comfy_entrypoint() -> JsonPromptExtension: + return JsonPromptExtension() diff --git a/comfy_extras/nodes_kandinsky5.py b/comfy_extras/nodes_kandinsky5.py index 015965498..96cca0386 100644 --- a/comfy_extras/nodes_kandinsky5.py +++ b/comfy_extras/nodes_kandinsky5.py @@ -13,7 +13,7 @@ class Kandinsky5ImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Kandinsky5ImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/kandinsky", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -71,7 +71,7 @@ class NormalizeVideoLatentStart(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="NormalizeVideoLatentStart", - category="model/conditioning/video_models", + category="model/conditioning", description="Normalizes the initial frames of a video latent to match the mean and standard deviation of subsequent reference frames. Helps reduce differences between the starting frames and the rest of the video.", inputs=[ io.Latent.Input("latent"), @@ -104,8 +104,9 @@ class CLIPTextEncodeKandinsky5(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeKandinsky5", + display_name="CLIP Text Encode (Kandinsky 5)", search_aliases=["kandinsky prompt"], - category="advanced/conditioning/kandinsky5", + category="model/conditioning/kandinsky", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_latent.py b/comfy_extras/nodes_latent.py index 32da9e8ac..1f93e34d6 100644 --- a/comfy_extras/nodes_latent.py +++ b/comfy_extras/nodes_latent.py @@ -262,6 +262,7 @@ class LatentBatch(io.ComfyNode): return io.Schema( node_id="LatentBatch", search_aliases=["combine latents", "merge latents", "join latents"], + display_name="Batch Latents (DEPRECATED)", category="model/latent/batch", is_deprecated=True, inputs=[ @@ -447,6 +448,7 @@ class ReplaceVideoLatentFrames(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ReplaceVideoLatentFrames", + display_name="Replace Video Latent Frames", category="model/latent/batch", inputs=[ io.Latent.Input("destination", tooltip="The destination latent where frames will be replaced."), diff --git a/comfy_extras/nodes_load_3d.py b/comfy_extras/nodes_load_3d.py index b339dc4ff..106b01f9d 100644 --- a/comfy_extras/nodes_load_3d.py +++ b/comfy_extras/nodes_load_3d.py @@ -52,15 +52,19 @@ class Load3D(IO.ComfyNode): ) @classmethod - def execute(cls, model_file, image, **kwargs) -> IO.NodeOutput: - image_path = folder_paths.get_annotated_filepath(image['image']) - mask_path = folder_paths.get_annotated_filepath(image['mask']) - normal_path = folder_paths.get_annotated_filepath(image['normal']) + def validate_inputs(cls, model_file, **kwargs) -> bool | str: + if not model_file or model_file == "none": + return True + if not folder_paths.exists_annotated_filepath(model_file): + return f"Invalid 3D model file: {model_file}" + return True + @classmethod + def execute(cls, model_file, image, **kwargs) -> IO.NodeOutput: load_image_node = nodes.LoadImage() - output_image, ignore_mask = load_image_node.load_image(image=image_path) - ignore_image, output_mask = load_image_node.load_image(image=mask_path) - normal_image, ignore_mask2 = load_image_node.load_image(image=normal_path) + output_image, ignore_mask = load_image_node.load_image(image=image['image']) + ignore_image, output_mask = load_image_node.load_image(image=image['mask']) + normal_image, ignore_mask2 = load_image_node.load_image(image=image['normal']) video = None @@ -88,6 +92,7 @@ class Preview3D(IO.ComfyNode): search_aliases=["view mesh", "3d viewer"], display_name="Preview 3D & Animation", category="3d", + description="Preview a 3D model file without saving it to the ComfyUI output directory.", is_experimental=True, is_output_node=True, inputs=[ @@ -132,11 +137,12 @@ class Preview3DAdvanced(IO.ComfyNode): display_name="Preview 3D (Advanced)", search_aliases=["preview 3d", "3d viewer", "view mesh", "frame 3d", "3d camera output"], category="3d", + description="Preview a 3D model file without saving it to the ComfyUI output directory.", is_experimental=True, is_output_node=True, inputs=[ IO.MultiType.Input( - "model_file", + "model_3d", types=[ IO.File3DGLB, IO.File3DGLTF, @@ -148,47 +154,245 @@ class Preview3DAdvanced(IO.ComfyNode): ], tooltip="3D model file from an upstream 3D node.", ), - IO.Load3D.Input("image"), - IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3D.Input("viewport_state"), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), IO.Int.Input("width", default=1024, min=1, max=4096, step=1), IO.Int.Input("height", default=1024, min=1, max=4096, step=1), ], outputs=[ - IO.File3DAny.Output(display_name="model_file"), - IO.Load3DCamera.Output(display_name="camera_info"), + IO.File3DAny.Output(display_name="model_3d"), IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), IO.Int.Output(display_name="width"), IO.Int.Output(display_name="height"), ], ) @classmethod - def execute(cls, model_file: Types.File3D, image, width: int, height: int, **kwargs) -> IO.NodeOutput: - filename = f"preview3d_advanced_{uuid.uuid4().hex}.{model_file.format}" - model_file.save_to(os.path.join(folder_paths.get_output_directory(), filename)) + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + filename = f"preview3d_advanced_{uuid.uuid4().hex}.{model_3d.format}" + model_3d.save_to(os.path.join(folder_paths.get_temp_directory(), filename)) + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} camera_info_input = kwargs.get("camera_info", None) - camera_info = camera_info_input if camera_info_input is not None else image['camera_info'] + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') model_3d_info_input = kwargs.get("model_3d_info", None) - model_3d_info = model_3d_info_input if model_3d_info_input is not None else image.get('model_3d_info', []) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) return IO.NodeOutput( - model_file, - camera_info, + model_3d, model_3d_info, + camera_info, width, height, ui=UI.PreviewUI3DAdvanced(filename, camera_info, model_3d_info), ) +class PreviewGaussianSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PreviewGaussianSplat", + display_name="Preview Splat", + category="3d", + description="Preview a gaussian splat 3D file without saving it to the ComfyUI output directory.", + is_experimental=True, + is_output_node=True, + search_aliases=[ + "view splat", + "view gaussian", + "view gaussian splat", + "preview gaussian", + "preview gaussian splat", + "view 3dgs", + "preview 3dgs", + "preview ply", + "preview spz", + "preview splat", + "preview ksplat", + ], + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DSplatAny, + IO.File3DPLY, + IO.File3DSPLAT, + IO.File3DSPZ, + IO.File3DKSPLAT, + ], + tooltip="A gaussian splat 3D file.", + ), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3D.Input("viewport_state"), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DSplatAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + filename = f"preview_splat_{uuid.uuid4().hex}.{model_3d.format}" + model_3d.save_to(os.path.join(folder_paths.get_temp_directory(), filename)) + + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + camera_info_input = kwargs.get("camera_info", None) + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') + model_3d_info_input = kwargs.get("model_3d_info", None) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) + return IO.NodeOutput( + model_3d, + model_3d_info, + camera_info, + width, + height, + ui=UI.PreviewUI3DAdvanced(filename, camera_info, model_3d_info), + ) + + +class PreviewPointCloud(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PreviewPointCloud", + display_name="Preview Point Cloud", + category="3d", + description="Preview a point cloud 3D file without saving it to the ComfyUI output directory.", + is_experimental=True, + is_output_node=True, + search_aliases=[ + "view point cloud", + "view pointcloud", + "preview point cloud", + "preview pointcloud", + "preview ply", + ], + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DPointCloudAny, + IO.File3DPLY, + ], + tooltip="Point cloud file (.ply)", + ), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3D.Input("viewport_state"), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DPointCloudAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + filename = f"preview_pointcloud_{uuid.uuid4().hex}.{model_3d.format}" + model_3d.save_to(os.path.join(folder_paths.get_temp_directory(), filename)) + + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + camera_info_input = kwargs.get("camera_info", None) + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') + model_3d_info_input = kwargs.get("model_3d_info", None) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) + return IO.NodeOutput( + model_3d, + model_3d_info, + camera_info, + width, + height, + ui=UI.PreviewUI3DAdvanced(filename, camera_info, model_3d_info), + ) + + +MESH_EXTENSIONS = {'.gltf', '.glb', '.obj', '.fbx', '.stl'} + + +class Load3DAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + input_dir = os.path.join(folder_paths.get_input_directory(), "3d") + os.makedirs(input_dir, exist_ok=True) + + input_path = Path(input_dir) + base_path = Path(folder_paths.get_input_directory()) + + files = [ + normalize_path(str(file_path.relative_to(base_path))) + for file_path in input_path.rglob("*") + if file_path.suffix.lower() in MESH_EXTENSIONS + ] + return IO.Schema( + node_id="Load3DAdvanced", + display_name="Load 3D (Advanced)", + category="3d", + search_aliases=[ + "load mesh", + "load gltf", + "load glb", + "load obj", + "load fbx", + "load stl", + ], + is_experimental=True, + inputs=[ + IO.Combo.Input("model_file", options=["none"] + sorted(files), upload=IO.UploadType.model), + IO.Load3D.Input("viewport_state"), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def validate_inputs(cls, model_file, **kwargs) -> bool | str: + if not model_file or model_file == "none": + return True + if not folder_paths.exists_annotated_filepath(model_file): + return f"Invalid 3D model file: {model_file}" + return True + + @classmethod + def execute(cls, model_file, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + file_3d = None + if model_file and model_file != "none": + file_3d = Types.File3D(folder_paths.get_annotated_filepath(model_file)) + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + model_3d_info = viewport_state.get('model_3d_info', []) + return IO.NodeOutput(file_3d, model_3d_info, viewport_state.get('camera_info'), width, height) + + class Load3DExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: return [ Load3D, + Load3DAdvanced, Preview3D, Preview3DAdvanced, + PreviewGaussianSplat, + PreviewPointCloud, ] diff --git a/comfy_extras/nodes_logic.py b/comfy_extras/nodes_logic.py index 95f6ab848..13c1685f7 100644 --- a/comfy_extras/nodes_logic.py +++ b/comfy_extras/nodes_logic.py @@ -89,7 +89,8 @@ class SwitchNode(io.ComfyNode): template = io.MatchType.Template("switch") return io.Schema( node_id="ComfySwitchNode", - display_name="Switch", + search_aliases=["if", "then", "switch", "conditional", "branch"], + display_name="If/Else Switch", category="utilities/logic", is_experimental=True, inputs=[ diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.py index 6d6078abe..85d76ecef 100644 --- a/comfy_extras/nodes_lt.py +++ b/comfy_extras/nodes_lt.py @@ -25,7 +25,7 @@ class GetICLoRAParameters(io.ComfyNode): display_name="Get IC-LoRA Parameters", description="Extracts IC-LoRA parameters from the safetensors metadata of a LoRA-loaded " "model and outputs them for LTXVAddGuide (eg. reference_downscale_factor).", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", search_aliases=["ic-lora", "ic lora", "iclora", "downscale factor", "reference downscale"], inputs=[ io.Model.Input( @@ -62,7 +62,7 @@ class EmptyLTXVLatentVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyLTXVLatentVideo", - category="model/latent/video/ltxv", + category="model/latent/ltxv", inputs=[ io.Int.Input("width", default=768, min=64, max=nodes.MAX_RESOLUTION, step=32), io.Int.Input("height", default=512, min=64, max=nodes.MAX_RESOLUTION, step=32), @@ -86,7 +86,7 @@ class LTXVImgToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVImgToVideo", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -131,7 +131,7 @@ class LTXVImgToVideoInplace(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVImgToVideoInplace", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Vae.Input("vae"), io.Image.Input("image"), @@ -251,7 +251,7 @@ class LTXVAddGuide(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVAddGuide", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -498,7 +498,7 @@ class LTXVCropGuides(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVCropGuides", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -542,7 +542,7 @@ class LTXVConditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVConditioning", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -566,7 +566,7 @@ class ModelSamplingLTXV(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ModelSamplingLTXV", - category="advanced/model", + category="model/patch/ltxv", inputs=[ io.Model.Input("model"), io.Float.Input("max_shift", default=2.05, min=0.0, max=100.0, step=0.01), @@ -746,7 +746,7 @@ class LTXVConcatAVLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVConcatAVLatent", - category="model/latent/video/ltxv", + category="model/latent/ltxv", inputs=[ io.Latent.Input("video_latent"), io.Latent.Input("audio_latent"), @@ -781,7 +781,7 @@ class LTXVSeparateAVLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVSeparateAVLatent", - category="model/latent/video/ltxv", + category="model/latent/ltxv", description="LTXV Separate AV Latent", inputs=[ io.Latent.Input("av_latent"), @@ -814,7 +814,7 @@ class LTXVReferenceAudio(io.ComfyNode): return io.Schema( node_id="LTXVReferenceAudio", display_name="LTXV Reference Audio (ID-LoRA)", - category="model/conditioning/audio", + category="model/conditioning/ltxv", description="Set reference audio for ID-LoRA speaker identity transfer. Encodes a reference audio clip into the conditioning and optionally patches the model with identity guidance (extra forward pass without reference, amplifying the speaker identity effect).", inputs=[ io.Model.Input("model"), diff --git a/comfy_extras/nodes_lt_audio.py b/comfy_extras/nodes_lt_audio.py index 052186083..2d774a0a3 100644 --- a/comfy_extras/nodes_lt_audio.py +++ b/comfy_extras/nodes_lt_audio.py @@ -40,7 +40,7 @@ class LTXVAudioVAEEncode(VAEEncodeAudio): return io.Schema( node_id="LTXVAudioVAEEncode", display_name="LTXV Audio VAE Encode", - category="model/latent/audio", + category="model/latent/ltxv", inputs=[ io.Audio.Input("audio", tooltip="The audio to be encoded."), io.Vae.Input( @@ -63,7 +63,7 @@ class LTXVAudioVAEDecode(io.ComfyNode): return io.Schema( node_id="LTXVAudioVAEDecode", display_name="LTXV Audio VAE Decode", - category="model/latent/audio", + category="model/latent/ltxv", inputs=[ io.Latent.Input("samples", tooltip="The latent to be decoded."), io.Vae.Input( @@ -96,7 +96,7 @@ class LTXVEmptyLatentAudio(io.ComfyNode): return io.Schema( node_id="LTXVEmptyLatentAudio", display_name="LTXV Empty Latent Audio", - category="model/latent/audio", + category="model/latent/ltxv", inputs=[ io.Int.Input( "frames_number", @@ -168,9 +168,9 @@ class LTXAVTextEncoderLoader(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="LTXAVTextEncoderLoader", - display_name="LTXV Audio Text Encoder Loader", - category="advanced/loaders", - description="[Recipes]\n\nltxav: gemma 3 12B", + display_name="Load LTXV Audio Text Encoder", + category="model/loaders", + description="Recipes:\nltxav: gemma 3 12B", inputs=[ io.Combo.Input( "text_encoder", diff --git a/comfy_extras/nodes_lt_upsampler.py b/comfy_extras/nodes_lt_upsampler.py index be9a36e69..ef36109d1 100644 --- a/comfy_extras/nodes_lt_upsampler.py +++ b/comfy_extras/nodes_lt_upsampler.py @@ -13,7 +13,7 @@ class LTXVLatentUpsampler(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="LTXVLatentUpsampler", - category="model/latent/video", + category="model/latent/ltxv", is_experimental=True, inputs=[ IO.Latent.Input("samples"), diff --git a/comfy_extras/nodes_lumina2.py b/comfy_extras/nodes_lumina2.py index c060a86a0..bc543c242 100644 --- a/comfy_extras/nodes_lumina2.py +++ b/comfy_extras/nodes_lumina2.py @@ -9,7 +9,7 @@ class RenormCFG(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="RenormCFG", - category="advanced/model", + category="model/patch", inputs=[ io.Model.Input("model"), io.Float.Input("cfg_trunc", default=100, min=0.0, max=100.0, step=0.01, advanced=True), @@ -80,8 +80,8 @@ class CLIPTextEncodeLumina2(io.ComfyNode): return io.Schema( node_id="CLIPTextEncodeLumina2", search_aliases=["lumina prompt"], - display_name="CLIP Text Encode for Lumina2", - category="model/conditioning", + display_name="CLIP Text Encode (Lumina 2)", + category="model/conditioning/lumina", description="Encodes a system prompt and a user prompt using a CLIP model into an embedding " "that can be used to guide the diffusion model towards generating specific images.", inputs=[ diff --git a/comfy_extras/nodes_mask.py b/comfy_extras/nodes_mask.py index 52484697a..3fae7221f 100644 --- a/comfy_extras/nodes_mask.py +++ b/comfy_extras/nodes_mask.py @@ -53,6 +53,7 @@ class LatentCompositeMasked(IO.ComfyNode): return IO.Schema( node_id="LatentCompositeMasked", search_aliases=["overlay latent", "layer latent", "paste latent", "inpaint latent"], + display_name="Latent Composite Masked", category="model/latent", inputs=[ IO.Latent.Input("destination"), @@ -418,17 +419,18 @@ class MaskPreview(IO.ComfyNode): search_aliases=["show mask", "view mask", "inspect mask", "debug mask"], display_name="Preview Mask", category="image/mask", - description="Saves the input images to your ComfyUI output directory.", + description="Preview the masks without saving them to the ComfyUI output directory.", inputs=[ IO.Mask.Input("mask"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Mask.Output(display_name="mask")] ) @classmethod def execute(cls, mask, filename_prefix="ComfyUI") -> IO.NodeOutput: - return IO.NodeOutput(ui=UI.PreviewMask(mask)) + return IO.NodeOutput(mask, ui=UI.PreviewMask(mask)) class MaskExtension(ComfyExtension): diff --git a/comfy_extras/nodes_mochi.py b/comfy_extras/nodes_mochi.py index 3dcea6ab3..3aaf23e69 100644 --- a/comfy_extras/nodes_mochi.py +++ b/comfy_extras/nodes_mochi.py @@ -10,7 +10,7 @@ class EmptyMochiLatentVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyMochiLatentVideo", - category="model/latent/video", + category="model/latent/mochi", inputs=[ io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.py index b27ac1296..a336ba079 100644 --- a/comfy_extras/nodes_model_advanced.py +++ b/comfy_extras/nodes_model_advanced.py @@ -59,7 +59,7 @@ class ModelSamplingDiscrete: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, sampling, zsnr): m = model.clone() @@ -97,7 +97,7 @@ class ModelSamplingStableCascade: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch/stable cascade" def patch(self, model, shift): m = model.clone() @@ -123,7 +123,7 @@ class ModelSamplingSD3: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch/stable diffusion" def patch(self, model, shift, multiplier=1000): m = model.clone() @@ -150,6 +150,7 @@ class ModelSamplingAuraFlow(ModelSamplingSD3): }} FUNCTION = "patch_aura" + CATEGORY = "model/patch" def patch_aura(self, model, shift): return self.patch(model, shift, multiplier=1.0) @@ -167,7 +168,7 @@ class ModelSamplingFlux: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch/flux" def patch(self, model, max_shift, base_shift, width, height): m = model.clone() @@ -202,7 +203,7 @@ class ModelSamplingContinuousEDM: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, sampling, sigma_max, sigma_min): m = model.clone() @@ -247,7 +248,7 @@ class ModelSamplingContinuousV: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, sampling, sigma_max, sigma_min): m = model.clone() @@ -273,7 +274,7 @@ class RescaleCFG: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, multiplier): def rescale_cfg(args): @@ -314,7 +315,7 @@ class ModelNoiseScale: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, noise_scale): m = model.clone() @@ -337,7 +338,7 @@ class ModelComputeDtype: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/debug/model" + CATEGORY = "advanced/debug" def patch(self, model, dtype): m = model.clone() diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index b6b29e34a..962d2a0bb 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -21,7 +21,7 @@ class ModelMergeSimple: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2, ratio): m = model1.clone() @@ -40,7 +40,7 @@ class ModelSubtract: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2, multiplier): m = model1.clone() @@ -58,7 +58,7 @@ class ModelAdd: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2): m = model1.clone() @@ -78,7 +78,7 @@ class CLIPMergeSimple: RETURN_TYPES = ("CLIP",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, clip1, clip2, ratio): m = clip1.clone() @@ -101,7 +101,7 @@ class CLIPSubtract: RETURN_TYPES = ("CLIP",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, clip1, clip2, multiplier): m = clip1.clone() @@ -123,7 +123,7 @@ class CLIPAdd: RETURN_TYPES = ("CLIP",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, clip1, clip2): m = clip1.clone() @@ -147,7 +147,7 @@ class ModelMergeBlocks: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2, **kwargs): m = model1.clone() @@ -242,7 +242,7 @@ class CheckpointSave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, model, clip, vae, filename_prefix, prompt=None, extra_pnginfo=None): save_checkpoint(model, clip=clip, vae=vae, filename_prefix=filename_prefix, output_dir=self.output_dir, prompt=prompt, extra_pnginfo=extra_pnginfo) @@ -261,7 +261,7 @@ class CLIPSave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, clip, filename_prefix, prompt=None, extra_pnginfo=None): prompt_info = "" @@ -318,7 +318,7 @@ class VAESave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, vae, filename_prefix, prompt=None, extra_pnginfo=None): full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) @@ -353,7 +353,7 @@ class ModelSave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, model, filename_prefix, prompt=None, extra_pnginfo=None): save_checkpoint(model, filename_prefix=filename_prefix, output_dir=self.output_dir, prompt=prompt, extra_pnginfo=extra_pnginfo) diff --git a/comfy_extras/nodes_model_merging_model_specific.py b/comfy_extras/nodes_model_merging_model_specific.py index 55eb3ccfe..e563d950b 100644 --- a/comfy_extras/nodes_model_merging_model_specific.py +++ b/comfy_extras/nodes_model_merging_model_specific.py @@ -1,7 +1,7 @@ import comfy_extras.nodes_model_merging class ModelMergeSD1(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): arg_dict = { "model1": ("MODEL",), @@ -27,7 +27,7 @@ class ModelMergeSD1(comfy_extras.nodes_model_merging.ModelMergeBlocks): class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -53,7 +53,7 @@ class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeSD3_2B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -77,7 +77,7 @@ class ModelMergeSD3_2B(comfy_extras.nodes_model_merging.ModelMergeBlocks): class ModelMergeAuraflow(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -104,7 +104,7 @@ class ModelMergeAuraflow(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeFlux1(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -130,7 +130,7 @@ class ModelMergeFlux1(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeSD35_Large(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -153,7 +153,7 @@ class ModelMergeSD35_Large(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeMochiPreview(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -175,7 +175,7 @@ class ModelMergeMochiPreview(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeLTXV(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -197,7 +197,7 @@ class ModelMergeLTXV(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeCosmos7B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -221,7 +221,7 @@ class ModelMergeCosmos7B(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeCosmos14B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -245,7 +245,7 @@ class ModelMergeCosmos14B(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeWAN2_1(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" DESCRIPTION = "1.3B model has 30 blocks, 14B model has 40 blocks. Image to video model has the extra img_emb." @classmethod @@ -269,7 +269,7 @@ class ModelMergeWAN2_1(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeCosmosPredict2_2B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -292,7 +292,7 @@ class ModelMergeCosmosPredict2_2B(comfy_extras.nodes_model_merging.ModelMergeBlo return {"required": arg_dict} class ModelMergeCosmosPredict2_14B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -315,7 +315,7 @@ class ModelMergeCosmosPredict2_14B(comfy_extras.nodes_model_merging.ModelMergeBl return {"required": arg_dict} class ModelMergeQwenImage(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -337,6 +337,36 @@ class ModelMergeQwenImage(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} +class ModelMergeKrea2(comfy_extras.nodes_model_merging.ModelMergeBlocks): + CATEGORY = "model/merging/model specific" + + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["first."] = argument + arg_dict["tmlp."] = argument + arg_dict["txtmlp."] = argument + arg_dict["tproj."] = argument + + for i in range(2): + arg_dict["txtfusion.layerwise_blocks.{}.".format(i)] = argument + + arg_dict["txtfusion.projector."] = argument + + for i in range(2): + arg_dict["txtfusion.refiner_blocks.{}.".format(i)] = argument + + for i in range(28): + arg_dict["blocks.{}.".format(i)] = argument + + arg_dict["last."] = argument + + return {"required": arg_dict} + NODE_CLASS_MAPPINGS = { "ModelMergeSD1": ModelMergeSD1, "ModelMergeSD2": ModelMergeSD1, #SD1 and SD2 have the same blocks @@ -353,4 +383,5 @@ NODE_CLASS_MAPPINGS = { "ModelMergeCosmosPredict2_2B": ModelMergeCosmosPredict2_2B, "ModelMergeCosmosPredict2_14B": ModelMergeCosmosPredict2_14B, "ModelMergeQwenImage": ModelMergeQwenImage, + "ModelMergeKrea2": ModelMergeKrea2, } diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py index bdccbf8c4..3f785c8b5 100644 --- a/comfy_extras/nodes_model_patch.py +++ b/comfy_extras/nodes_model_patch.py @@ -232,7 +232,7 @@ class ModelPatchLoader: FUNCTION = "load_model_patch" EXPERIMENTAL = True - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" def load_model_patch(self, name): model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name) @@ -479,7 +479,7 @@ class QwenImageDiffsynthControlnet: FUNCTION = "diffsynth_controlnet" EXPERIMENTAL = True - CATEGORY = "advanced/loaders/qwen" + CATEGORY = "model/patch/qwen" def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None): model_patched = model.clone() @@ -512,7 +512,7 @@ class ZImageFunControlnet(QwenImageDiffsynthControlnet): }, "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",)}} - CATEGORY = "advanced/loaders/zimage" + CATEGORY = "model/patch/z-image" class UsoStyleProjectorPatch: def __init__(self, model_patch, encoded_image): @@ -675,3 +675,11 @@ NODE_CLASS_MAPPINGS = { "USOStyleReference": USOStyleReference, "SUPIRApply": SUPIRApply, } + +NODE_DISPLAY_NAME_MAPPINGS = { + "ModelPatchLoader": "Load Model Patch", + "QwenImageDiffsynthControlnet": "Apply Qwen Image DiffSynth ControlNet", + "ZImageFunControlnet": "Apply Z-Image Fun ControlNet", + "USOStyleReference": "Apply USO Style Reference", + "SUPIRApply": "Apply SUPIR Patch", +} diff --git a/comfy_extras/nodes_moge.py b/comfy_extras/nodes_moge.py index 422949531..a63f0414b 100644 --- a/comfy_extras/nodes_moge.py +++ b/comfy_extras/nodes_moge.py @@ -8,6 +8,7 @@ import folder_paths from comfy_api.latest import ComfyExtension, Types, io from typing_extensions import override +from comfy.ldm.colormap import turbo as _turbo from comfy.ldm.moge.model import MoGeModel from comfy.ldm.moge.geometry import triangulate_grid_mesh from comfy.ldm.moge.panorama import get_panorama_cameras, split_panorama_image, merge_panorama_depth, spherical_uv_to_directions, _uv_grid @@ -27,19 +28,6 @@ MoGeGeometry = io.Custom("MOGE_GEOMETRY") # "image": torch.Tensor (B, H, W, 3) in [0, 1], CPU (always present) -def _turbo(x: torch.Tensor) -> torch.Tensor: - """Anton Mikhailov polynomial approximation of the turbo colormap.""" - x = x.clamp(0.0, 1.0) - x2 = x * x - x3 = x2 * x - x4 = x2 * x2 - x5 = x4 * x - r = 0.13572138 + 4.61539260*x - 42.66032258*x2 + 132.13108234*x3 - 152.94239396*x4 + 59.28637943*x5 - g = 0.09140261 + 2.19418839*x + 4.84296658*x2 - 14.18503333*x3 + 4.27729857*x4 + 2.82956604*x5 - b = 0.10667330 + 12.64194608*x - 60.58204836*x2 + 110.36276771*x3 - 89.90310912*x4 + 27.34824973*x5 - return torch.stack([r, g, b], dim=-1).clamp(0.0, 1.0) - - def _normals_from_points(points: torch.Tensor) -> torch.Tensor: """Camera-space surface normals from a (B, H, W, 3) point map (v1 fallback).""" finite = torch.isfinite(points).all(dim=-1) diff --git a/comfy_extras/nodes_photomaker.py b/comfy_extras/nodes_photomaker.py index 8a2248572..72fad1673 100644 --- a/comfy_extras/nodes_photomaker.py +++ b/comfy_extras/nodes_photomaker.py @@ -123,7 +123,8 @@ class PhotoMakerLoader(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="PhotoMakerLoader", - category="experimental/photomaker", + display_name="Load PhotoMaker Model", + category="model/loaders", inputs=[ io.Combo.Input("photomaker_model_name", options=folder_paths.get_filename_list("photomaker")), ], @@ -149,7 +150,8 @@ class PhotoMakerEncode(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="PhotoMakerEncode", - category="experimental/photomaker", + display_name="PhotoMaker Encode", + category="model/conditioning/photomaker", inputs=[ io.Photomaker.Input("photomaker"), io.Image.Input("image"), diff --git a/comfy_extras/nodes_pid.py b/comfy_extras/nodes_pid.py index 811b9ae8e..a3ffd9671 100644 --- a/comfy_extras/nodes_pid.py +++ b/comfy_extras/nodes_pid.py @@ -14,15 +14,13 @@ class PiDConditioning(io.ComfyNode): return io.Schema( node_id="PiDConditioning", display_name="PiD Conditioning", - category="advanced/conditioning", - description=( - "Attaches a latent and a degrade_sigma scalar to a CONDITIONING for PiD decoding/upscaling" - ), + category="model/conditioning", + description=("Attaches a latent and a degrade_sigma scalar to a CONDITIONING for PiD decoding/upscaling"), inputs=[ io.Conditioning.Input("positive"), io.Latent.Input("latent", tooltip="latent (from VAEEncode or a KSampler)."), - io.Combo.Input("latent_format", options=["flux", "sd3"], default="flux", - tooltip="Flux1 and Flux2 latents auto-detected from channel dim, sd3 has to be selected manually."), + io.Combo.Input("latent_format", options=["flux", "sd3", "sdxl", "qwenimage"], default="flux", + tooltip="Flux1 (16-ch) and Flux2 (128-ch) latents are auto-detected from channel dim under 'flux'. For SD3 (16-ch), SDXL (4-ch), or QwenImage (16-ch), select manually."), io.Float.Input( "degrade_sigma", default=0.0, min=0.0, max=1.0, step=0.01, tooltip="0 = clean latent. Increase to denoise corrupted latent outputs.", @@ -36,9 +34,17 @@ class PiDConditioning(io.ComfyNode): samples = latent["samples"] if latent_format == "flux": fmt_cls = comfy.latent_formats.Flux2 if samples.shape[1] == 128 else comfy.latent_formats.Flux - else: + elif latent_format == "sd3": fmt_cls = comfy.latent_formats.SD3 + elif latent_format == "sdxl": + fmt_cls = comfy.latent_formats.SDXL + elif latent_format == "qwenimage": + fmt_cls = comfy.latent_formats.Wan21 + else: + raise ValueError(f"Unknown latent_format: {latent_format}") lq_latent = fmt_cls().process_in(samples) + if lq_latent.ndim == 5: + lq_latent = lq_latent[:, :, 0] sigma_t = torch.tensor([float(degrade_sigma)], dtype=torch.float32) return io.NodeOutput(node_helpers.conditioning_set_values( positive, {"lq_latent": lq_latent, "degrade_sigma": sigma_t}, diff --git a/comfy_extras/nodes_pixart.py b/comfy_extras/nodes_pixart.py index 2f1b73e60..f878a33b5 100644 --- a/comfy_extras/nodes_pixart.py +++ b/comfy_extras/nodes_pixart.py @@ -7,8 +7,9 @@ class CLIPTextEncodePixArtAlpha(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodePixArtAlpha", + display_name="CLIP Text Encode (PixArt Alpha)", search_aliases=["pixart prompt"], - category="advanced/conditioning", + category="model/conditioning/pixart", description="Encodes text and sets the resolution conditioning for PixArt Alpha. Does not apply to PixArt Sigma.", inputs=[ io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py index 3e440433e..763b8a52f 100644 --- a/comfy_extras/nodes_post_processing.py +++ b/comfy_extras/nodes_post_processing.py @@ -616,7 +616,7 @@ class BatchLatentsNode(io.ComfyNode): node_id="BatchLatentsNode", search_aliases=["combine latents", "stack latents", "merge latents"], display_name="Batch Latents", - category="model/latent", + category="model/latent/batch", inputs=[ io.Autogrow.Input("latents", template=autogrow_template) ], diff --git a/comfy_extras/nodes_preview_any.py b/comfy_extras/nodes_preview_any.py index 1070a69d0..d985f3287 100644 --- a/comfy_extras/nodes_preview_any.py +++ b/comfy_extras/nodes_preview_any.py @@ -18,6 +18,7 @@ class PreviewAny(): CATEGORY = "utilities" SEARCH_ALIASES = ["show output", "inspect", "debug", "print value", "show text"] + DESCRIPTION = "Preview any input value as text." def main(self, source=None): torch.set_printoptions(edgeitems=6) diff --git a/comfy_extras/nodes_primitive.py b/comfy_extras/nodes_primitive.py index c44b09098..35761863f 100644 --- a/comfy_extras/nodes_primitive.py +++ b/comfy_extras/nodes_primitive.py @@ -10,12 +10,10 @@ class String(io.ComfyNode): return io.Schema( node_id="PrimitiveString", search_aliases=["text", "string", "text box", "prompt"], - display_name="Text String", + display_name="Text", category="utilities/primitive", - inputs=[ - io.String.Input("value"), - ], - outputs=[io.String.Output()], + inputs=[io.String.Input("value")], + outputs=[io.String.Output()] ) @classmethod @@ -29,12 +27,10 @@ class StringMultiline(io.ComfyNode): return io.Schema( node_id="PrimitiveStringMultiline", search_aliases=["text", "string", "text multiline", "string multiline", "text box", "prompt"], - display_name="Text String (Multiline)", + display_name="Text (Multiline)", category="utilities/primitive", essentials_category="Basics", - inputs=[ - io.String.Input("value", multiline=True), - ], + inputs=[io.String.Input("value", multiline=True)], outputs=[io.String.Output()], ) diff --git a/comfy_extras/nodes_qwen.py b/comfy_extras/nodes_qwen.py index 5b92814a4..4960774db 100644 --- a/comfy_extras/nodes_qwen.py +++ b/comfy_extras/nodes_qwen.py @@ -12,7 +12,7 @@ class TextEncodeQwenImageEdit(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeQwenImageEdit", - category="advanced/conditioning", + category="model/conditioning/qwen image", inputs=[ io.Clip.Input("clip"), io.String.Input("prompt", multiline=True, dynamic_prompts=True), @@ -55,7 +55,7 @@ class TextEncodeQwenImageEditPlus(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeQwenImageEditPlus", - category="advanced/conditioning", + category="model/conditioning/qwen image", inputs=[ io.Clip.Input("clip"), io.String.Input("prompt", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_resolution.py b/comfy_extras/nodes_resolution.py index dc405291c..083e47ae4 100644 --- a/comfy_extras/nodes_resolution.py +++ b/comfy_extras/nodes_resolution.py @@ -6,24 +6,24 @@ from comfy_api.latest import ComfyExtension, io class AspectRatio(str, Enum): SQUARE = "1:1 (Square)" + PHOTO_V = "2:3 (Portrait Photo)" PHOTO_H = "3:2 (Photo)" + STANDARD_V = "3:4 (Portrait Standard)" STANDARD_H = "4:3 (Standard)" + WIDESCREEN_V = "9:16 (Portrait Widescreen)" WIDESCREEN_H = "16:9 (Widescreen)" ULTRAWIDE_H = "21:9 (Ultrawide)" - PHOTO_V = "2:3 (Portrait Photo)" - STANDARD_V = "3:4 (Portrait Standard)" - WIDESCREEN_V = "9:16 (Portrait Widescreen)" ASPECT_RATIOS: dict[AspectRatio, tuple[int, int]] = { AspectRatio.SQUARE: (1, 1), + AspectRatio.PHOTO_V: (2, 3), AspectRatio.PHOTO_H: (3, 2), + AspectRatio.STANDARD_V: (3, 4), AspectRatio.STANDARD_H: (4, 3), + AspectRatio.WIDESCREEN_V: (9, 16), AspectRatio.WIDESCREEN_H: (16, 9), AspectRatio.ULTRAWIDE_H: (21, 9), - AspectRatio.PHOTO_V: (2, 3), - AspectRatio.STANDARD_V: (3, 4), - AspectRatio.WIDESCREEN_V: (9, 16), } @@ -50,26 +50,35 @@ class ResolutionSelector(io.ComfyNode): min=0.1, max=16.0, step=0.1, - tooltip="Target total megapixels. 1.0 MP ≈ 1024×1024 for square.", + tooltip="Target total megapixels. 1.0 MP ≈ 1024x1024 for square.", + ), + io.Int.Input( + id="multiple", + default=8, + min=8, + max=128, + step=4, + tooltip="Nearest multiple of the result to set the selected resolution to.", + advanced=True, ), ], outputs=[ io.Int.Output( - "width", tooltip="Calculated width in pixels (multiple of 8)." + "width", tooltip="Calculated width in pixels multiplied by the selected multiple." ), io.Int.Output( - "height", tooltip="Calculated height in pixels (multiple of 8)." + "height", tooltip="Calculated height in pixels multiplied by the selected multiple." ), ], ) @classmethod - def execute(cls, aspect_ratio: str, megapixels: float) -> io.NodeOutput: + def execute(cls, aspect_ratio: str, megapixels: float, multiple: int) -> io.NodeOutput: w_ratio, h_ratio = ASPECT_RATIOS[aspect_ratio] total_pixels = megapixels * 1024 * 1024 scale = math.sqrt(total_pixels / (w_ratio * h_ratio)) - width = round(w_ratio * scale / 8) * 8 - height = round(h_ratio * scale / 8) * 8 + width = round(w_ratio * scale / multiple) * multiple + height = round(h_ratio * scale / multiple) * multiple return io.NodeOutput(width, height) diff --git a/comfy_extras/nodes_rtdetr.py b/comfy_extras/nodes_rtdetr.py index e5a9b3902..653f3af2f 100644 --- a/comfy_extras/nodes_rtdetr.py +++ b/comfy_extras/nodes_rtdetr.py @@ -14,7 +14,7 @@ class RTDETR_detect(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="RTDETR_detect", - display_name="RT-DETR Detect", + display_name="Run Real-Time Detection (RT-DETR)", category="image/detection", search_aliases=["bbox", "bounding box", "object detection", "coco"], inputs=[ diff --git a/comfy_extras/nodes_sam3.py b/comfy_extras/nodes_sam3.py index daac52f9b..f88aec925 100644 --- a/comfy_extras/nodes_sam3.py +++ b/comfy_extras/nodes_sam3.py @@ -264,7 +264,7 @@ class SAM3_VideoTrack(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SAM3_VideoTrack", - display_name="SAM3 Video Track", + display_name="Run SAM3 Video Track", category="image/detection", search_aliases=["sam3", "video", "track", "propagate"], inputs=[ diff --git a/comfy_extras/nodes_save_3d.py b/comfy_extras/nodes_save_3d.py index a91549e7f..e9fd07326 100644 --- a/comfy_extras/nodes_save_3d.py +++ b/comfy_extras/nodes_save_3d.py @@ -13,7 +13,7 @@ from typing_extensions import override import folder_paths from comfy.cli_args import args -from comfy_api.latest import ComfyExtension, IO, Types +from comfy_api.latest import ComfyExtension, IO, Types, UI def pack_variable_mesh_batch(vertices, faces, colors=None, uvs=None, texture=None, unlit=False): @@ -337,6 +337,12 @@ class SaveGLB(IO.ComfyNode): IO.File3DFBX, IO.File3DSTL, IO.File3DUSDZ, + IO.File3DPLY, + IO.File3DSPLAT, + IO.File3DSPZ, + IO.File3DKSPLAT, + IO.File3DSplatAny, + IO.File3DPointCloudAny, IO.File3DAny, ], tooltip="Mesh or 3D file to save", @@ -400,10 +406,165 @@ class SaveGLB(IO.ComfyNode): return IO.NodeOutput(ui={"3d": results}) +def _save_file3d_to_output(model_3d: Types.File3D, filename_prefix: str) -> str: + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path( + filename_prefix, folder_paths.get_output_directory() + ) + ext = model_3d.format or "glb" + saved_filename = f"{filename}_{counter:05}.{ext}" + model_3d.save_to(os.path.join(full_output_folder, saved_filename)) + return f"{subfolder}/{saved_filename}" if subfolder else saved_filename + + +def execute_save_3d_advanced(model_3d, viewport_state, width, height, filename_prefix, kwargs) -> IO.NodeOutput: + model_file = _save_file3d_to_output(model_3d, filename_prefix) + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + camera_info_input = kwargs.get("camera_info", None) + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') + model_3d_info_input = kwargs.get("model_3d_info", None) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) + return IO.NodeOutput( + model_3d, + model_3d_info, + camera_info, + width, + height, + ui=UI.PreviewUI3DAdvanced(model_file, camera_info, model_3d_info), + ) + + +class Save3DAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="Save3DAdvanced", + display_name="Save 3D (Advanced)", + search_aliases=["save 3d", "export 3d model", "save mesh advanced"], + category="3d", + is_experimental=True, + is_output_node=True, + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DGLB, + IO.File3DGLTF, + IO.File3DFBX, + IO.File3DOBJ, + IO.File3DSTL, + IO.File3DUSDZ, + IO.File3DAny, + ], + tooltip="3D model file from an upstream 3D node.", + ), + IO.String.Input("filename_prefix", default="3d/ComfyUI"), + IO.Load3D.Input("viewport_state"), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, filename_prefix: str, **kwargs) -> IO.NodeOutput: + return execute_save_3d_advanced(model_3d, viewport_state, width, height, filename_prefix, kwargs) + + +class SaveGaussianSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveGaussianSplat", + display_name="Save Splat", + search_aliases=["save splat", "save gaussian splat", "export gaussian", "export splat"], + category="3d", + is_experimental=True, + is_output_node=True, + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DSplatAny, + IO.File3DPLY, + IO.File3DSPLAT, + IO.File3DSPZ, + IO.File3DKSPLAT, + ], + tooltip="A gaussian splat 3D file.", + ), + IO.String.Input("filename_prefix", default="3d/ComfyUI"), + IO.Load3D.Input("viewport_state"), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DSplatAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, filename_prefix: str, **kwargs) -> IO.NodeOutput: + return execute_save_3d_advanced(model_3d, viewport_state, width, height, filename_prefix, kwargs) + + +class SavePointCloud(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SavePointCloud", + display_name="Save Point Cloud", + search_aliases=["save point cloud", "save pointcloud", "export point cloud"], + category="3d", + is_experimental=True, + is_output_node=True, + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DPointCloudAny, + IO.File3DPLY, + ], + tooltip="Point cloud file (.ply)", + ), + IO.String.Input("filename_prefix", default="3d/ComfyUI"), + IO.Load3D.Input("viewport_state"), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DPointCloudAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, filename_prefix: str, **kwargs) -> IO.NodeOutput: + return execute_save_3d_advanced(model_3d, viewport_state, width, height, filename_prefix, kwargs) + + class Save3DExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: - return [SaveGLB] + return [SaveGLB, Save3DAdvanced, SaveGaussianSplat, SavePointCloud] async def comfy_entrypoint() -> Save3DExtension: diff --git a/comfy_extras/nodes_scail.py b/comfy_extras/nodes_scail.py new file mode 100644 index 000000000..55c9897e3 --- /dev/null +++ b/comfy_extras/nodes_scail.py @@ -0,0 +1,351 @@ +"""SCAIL / SCAIL-2 nodes: the WanSCAILToVideo conditioning node and the SAM3 +preprocessing that turns video tracks into the bundle the SCAIL-2 model consumes.""" + +from typing_extensions import override + +import torch +import torch.nn.functional as F + +import nodes +import node_helpers +import comfy.model_management +import comfy.utils +from comfy_api.latest import ComfyExtension, io +from comfy.ldm.sam3.tracker import unpack_masks + +SAM3TrackData = io.Custom("SAM3_TRACK_DATA") + + +# Model was trained on these exact colors; deviating degrades multi-identity quality. +DEFAULT_PALETTE = [ + (0.0, 0.0, 1.0), # Blue + (1.0, 0.0, 0.0), # Red + (0.0, 1.0, 0.0), # Green + (1.0, 0.0, 1.0), # Magenta + (0.0, 1.0, 1.0), # Cyan + (1.0, 1.0, 0.0), # Yellow +] + + +def _unpack(track_data): + packed = track_data["packed_masks"] + if packed is None or packed.shape[1] == 0: + return None + return unpack_masks(packed) + + +def _first_appearance_cx_area(masks_bool): + """Per object: first frame it appears in, plus centroid-x and area in that frame.""" + m = masks_bool.float() + T, H, W = m.shape[0], m.shape[-2], m.shape[-1] + grid_x = torch.arange(W, device=m.device, dtype=m.dtype).view(1, 1, 1, W) + area_t = m.sum(dim=(-1, -2)) + cx_t = (m * grid_x).sum(dim=(-1, -2)) / area_t.clamp(min=1) + present = area_t > 0 + frame_idx = torch.arange(T, device=m.device).unsqueeze(1) + first_t = torch.where(present, frame_idx, T).amin(dim=0) + sel = first_t.clamp(max=T - 1).unsqueeze(0) + cx = cx_t.gather(0, sel).squeeze(0) + area = area_t.gather(0, sel).squeeze(0) + return first_t.tolist(), (cx / W).tolist(), (area / (H * W)).tolist() + + +def _subset_track_data(track_data, obj_indices): + out = dict(track_data) + packed = track_data["packed_masks"] + if packed is None or not obj_indices: + out["packed_masks"] = None + if "scores" in out: + out["scores"] = [] + return out + out["packed_masks"] = packed[:, obj_indices].contiguous() + scores = track_data.get("scores") + if scores is not None: + out["scores"] = [scores[i] for i in obj_indices if i < len(scores)] + return out + + +def _render_colored_masks(track_data, background="black"): + packed = track_data["packed_masks"] + H, W = track_data["orig_size"] + device = comfy.model_management.intermediate_device() + dtype = comfy.model_management.intermediate_dtype() + bg_rgb = (1.0, 1.0, 1.0) if background.startswith("white") else (0.0, 0.0, 0.0) + if packed is None or packed.shape[1] == 0: + T = track_data.get("n_frames", 1) if packed is None else packed.shape[0] + out = torch.empty(T, H, W, 3, device=device, dtype=dtype) + out[..., 0], out[..., 1], out[..., 2] = bg_rgb[0], bg_rgb[1], bg_rgb[2] + return out + T, N_obj = packed.shape[0], packed.shape[1] + colors = torch.tensor( + [DEFAULT_PALETTE[i % len(DEFAULT_PALETTE)] for i in range(N_obj)], + device=device, dtype=dtype, + ) + masks_full = unpack_masks(packed.to(device)).float() + Hm, Wm = masks_full.shape[-2], masks_full.shape[-1] + masks_full = F.interpolate( + masks_full.view(T * N_obj, 1, Hm, Wm), size=(H, W), mode="nearest" + ).view(T, N_obj, H, W) > 0.5 + any_mask = masks_full.any(dim=1) + color_overlay = colors[masks_full.to(torch.uint8).argmax(dim=1)] + bg_tensor = torch.tensor(bg_rgb, device=device, dtype=color_overlay.dtype).view(1, 1, 1, 3) + return torch.where(any_mask.unsqueeze(-1), color_overlay, bg_tensor.expand_as(color_overlay)) + + +def _render_mask_as_identity(mask, background="black"): + """Plain comfy MASK (B,H,W) or (H,W) -> (B,H,W,3) rendered as a single identity (palette[0]) + on the given background. A batch is treated as multiple views of that one subject.""" + device = comfy.model_management.intermediate_device() + dtype = comfy.model_management.intermediate_dtype() + if mask.ndim == 2: + mask = mask.unsqueeze(0) + mask = mask.to(device=device, dtype=dtype) + B, H, W = mask.shape + bg_rgb = (1.0, 1.0, 1.0) if background.startswith("white") else (0.0, 0.0, 0.0) + color = torch.tensor(DEFAULT_PALETTE[0], device=device, dtype=dtype).view(1, 1, 1, 3) + bg = torch.tensor(bg_rgb, device=device, dtype=dtype).view(1, 1, 1, 3) + return torch.where((mask > 0.5).unsqueeze(-1), color.expand(B, H, W, 3), bg.expand(B, H, W, 3)) + + +def _extract_mask_to_28ch(rgb_video): + """Colored RGB mask (T, H, W, 3) in [0, 1] -> SCAIL-2 28-channel binary latent + (1, T_lat, 28, H_lat, W_lat). 7 per-color binary channels (white/r/g/b/y/m/c) + threshold-extracted at 225/255, 8x spatial downsample, 4-frame temporal stacking.""" + T, H, W, _ = rgb_video.shape + _ON_THRESH = 225.0 / 255.0 + mask = rgb_video.movedim(-1, 1).float() + R = (mask[:, 0:1] > _ON_THRESH).float() + G = (mask[:, 1:2] > _ON_THRESH).float() + B = (mask[:, 2:3] > _ON_THRESH).float() + nR, nG, nB = 1 - R, 1 - G, 1 - B + binary_7ch = torch.cat([ + R * G * B, # white + R * nG * nB, # red + nR * G * nB, # green + nR * nG * B, # blue + R * G * nB, # yellow + R * nG * B, # magenta + nR * G * B, # cyan + ], dim=1) + H_lat, W_lat = H, W + for _ in range(3): + H_lat = (H_lat + 1) // 2 + W_lat = (W_lat + 1) // 2 + binary_7ch = torch.nn.functional.interpolate(binary_7ch, size=(H_lat, W_lat), mode='area') + T_latent = (T - 1) // 4 + 1 + padded = torch.cat([binary_7ch[:1].repeat(4, 1, 1, 1), binary_7ch[1:]], dim=0) + out = padded.view(T_latent, 28, H_lat, W_lat) + return out.unsqueeze(0) + + +class WanSCAILToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="WanSCAILToVideo", + category="model/conditioning/wan/scail", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Int.Input("width", default=512, min=32, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("height", default=896, min=32, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("pose_video", optional=True, tooltip="Video used for pose conditioning. Will be downscaled to half the resolution of the main video."), + io.Image.Input("pose_video_mask", optional=True, tooltip="SCAIL-2 only. Colored per-identity SAM3 mask video at the same resolution as pose_video."), + io.Boolean.Input("replacement_mode", default=False, optional=True, tooltip="SCAIL-2 only. False = Animation Mode (pose_video_mask should have black background). True = Replacement Mode (pose_video_mask should have white background)."), + io.Float.Input("pose_strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Strength of the pose latent."), + io.Float.Input("pose_start", default=0.0, min=0.0, max=1.0, step=0.01, tooltip="Start step of the pose conditioning."), + io.Float.Input("pose_end", default=1.0, min=0.0, max=1.0, step=0.01, tooltip="End step of the pose conditioning."), + io.Image.Input("reference_image", optional=True, tooltip="Reference image. The first image is the primary reference (composite all identities onto it). SCAIL-2: extra batch images are used as additional views (back view, close-up, occluded background), each needing a matching reference_image_mask in that identity's color."), + io.Image.Input("reference_image_mask", optional=True, tooltip="SCAIL-2 only. Colored reference mask, batch matching reference_image (first = primary reference mask, rest = identity masks for the additional reference_image)."), + io.ClipVisionOutput.Input("clip_vision_output", optional=True, tooltip="CLIP vision features for conditioning. Model is trained with stretch resize to aspect ratio."), + io.Int.Input("video_frame_offset", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1, tooltip="Cumulative output frame this chunk begins at. Wire from the previous chunk's video_frame_offset output."), + io.Int.Input("previous_frame_count", default=5, min=1, max=nodes.MAX_RESOLUTION, step=4, tooltip="Tail frames of previous_frames to anchor. SCAIL-2 trained at 5 (81-frame chunks, 76-frame step)."), + io.Image.Input("previous_frames", optional=True, tooltip="SCAIL-2 only. Full decoded output of the previous chunk. Only the last previous_frame_count are used as the extension anchor."), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent", tooltip="Empty latent of the generation size."), + io.Int.Output(display_name="video_frame_offset", tooltip="Adjusted offset + length. Wire into the next chunk."), + ], + is_experimental=True, + ) + + @classmethod + def execute(cls, positive, negative, vae, width, height, length, batch_size, pose_strength, pose_start, pose_end, + video_frame_offset, previous_frame_count, replacement_mode=False, reference_image=None, clip_vision_output=None, pose_video=None, + pose_video_mask=None, reference_image_mask=None, previous_frames=None) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + noise_mask = None + + ref_mask_flag = not replacement_mode + positive = node_helpers.conditioning_set_values(positive, {"ref_mask_flag": ref_mask_flag}) + negative = node_helpers.conditioning_set_values(negative, {"ref_mask_flag": ref_mask_flag}) + + prev_trimmed = None + if previous_frames is not None and previous_frames.shape[0] > 0: + prev_trimmed = previous_frames[-previous_frame_count:] + video_frame_offset -= prev_trimmed.shape[0] + video_frame_offset = max(0, video_frame_offset) + + if reference_image is not None: + ref_imgs = comfy.utils.common_upscale(reference_image.movedim(-1, 1), width, height, "bicubic", "center").movedim(1, -1) + n_ref = ref_imgs.shape[0] + # SCAIL-2 multi-reference: the first image is the primary ref, the rest are additional references. + + # Replacement Mode: composite each ref on black bg using its mask as alpha matte + if replacement_mode and reference_image_mask is not None: + rm = comfy.utils.common_upscale(reference_image_mask.movedim(-1, 1), width, height, "nearest-exact", "center").movedim(1, -1) + rm = rm[[min(i, rm.shape[0] - 1) for i in range(n_ref)]] + is_char = (rm[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(ref_imgs.dtype) + ref_imgs = ref_imgs * is_char + # encode each ref individually so each stays a single latent frame (a batched encode would be treated as a video) + ref_latents = [vae.encode(ref_imgs[i:i + 1, :, :, :3]) for i in range(n_ref)] + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True) + + if clip_vision_output is not None: + positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) + negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) + + if pose_video is not None: + if pose_video.shape[0] <= video_frame_offset: + pose_video = None + else: + pose_video = pose_video[video_frame_offset:] + if pose_video_mask is not None: + if pose_video_mask.shape[0] <= video_frame_offset: + pose_video_mask = None + else: + pose_video_mask = pose_video_mask[video_frame_offset:] + + # Truncate pose+mask jointly to the shorter of the two, capped at length. + ts = [v.shape[0] for v in (pose_video, pose_video_mask) if v is not None] + if ts: + T_kept = ((min(min(ts), length) - 1) // 4) * 4 + 1 + if pose_video is not None: + pose_video = pose_video[:T_kept] + if pose_video_mask is not None: + pose_video_mask = pose_video_mask[:T_kept] + + if pose_video is not None: + pose_video = comfy.utils.common_upscale(pose_video[:length].movedim(-1, 1), width // 2, height // 2, "area", "center").movedim(1, -1) + pose_video_latent = vae.encode(pose_video[:, :, :, :3]) * pose_strength + positive = node_helpers.conditioning_set_values_with_timestep_range(positive, {"pose_video_latent": pose_video_latent}, pose_start, pose_end) + negative = node_helpers.conditioning_set_values_with_timestep_range(negative, {"pose_video_latent": pose_video_latent}, pose_start, pose_end) + + if pose_video_mask is not None: + mask_video_hw = comfy.utils.common_upscale(pose_video_mask[:length].movedim(-1, 1), width // 2, height // 2, "area", "center").movedim(1, -1) + driving_mask_28ch = _extract_mask_to_28ch(mask_video_hw) + positive = node_helpers.conditioning_set_values(positive, {"driving_mask_28ch": driving_mask_28ch}) + negative = node_helpers.conditioning_set_values(negative, {"driving_mask_28ch": driving_mask_28ch}) + + # The ref mask binds reference frames to identities, so it only applies when there's a reference image. + if reference_image_mask is not None and reference_image is not None: + ref_mask_hw = comfy.utils.common_upscale(reference_image_mask.movedim(-1, 1), width, height, "nearest-exact", "center").movedim(1, -1) + n_masks = ref_mask_hw.shape[0] + n_ref = reference_image.shape[0] + + add_masks = [_extract_mask_to_28ch(ref_mask_hw[min(i, n_masks - 1)][None]) for i in range(1, n_ref)] + ref_mask_1f = _extract_mask_to_28ch(ref_mask_hw[:1]) + zeros = torch.zeros((1, latent.shape[2], 28, ref_mask_1f.shape[-2], ref_mask_1f.shape[-1]), device=ref_mask_1f.device, dtype=ref_mask_1f.dtype) + ref_mask_28ch = torch.cat(add_masks + [ref_mask_1f, zeros], dim=1) + positive = node_helpers.conditioning_set_values(positive, {"ref_mask_28ch": ref_mask_28ch}) + negative = node_helpers.conditioning_set_values(negative, {"ref_mask_28ch": ref_mask_28ch}) + + if prev_trimmed is not None: + pf = comfy.utils.common_upscale(prev_trimmed.movedim(-1, 1), width, height, "bicubic", "center").movedim(1, -1) + prev_latent = vae.encode(pf[:, :, :, :3]) + prev_latent_frames = min(prev_latent.shape[2], latent.shape[2]) + latent[:, :, :prev_latent_frames] = prev_latent[:, :, :prev_latent_frames].to(latent.dtype) + noise_mask = torch.ones((1, 1, latent.shape[2], latent.shape[-2], latent.shape[-1]), device=latent.device, dtype=latent.dtype) + noise_mask[:, :, :prev_latent_frames] = 0.0 + + out_latent = {"samples": latent} + if noise_mask is not None: + out_latent["noise_mask"] = noise_mask + return io.NodeOutput(positive, negative, out_latent, video_frame_offset + length) + + +class SCAIL2ColoredMask(io.ComfyNode): + """Render SAM3 tracks for the driving pose video and reference image(s) into the + colored masks WanSCAILToVideo consumes. Shared `sort_by` keeps each identity on the + same color across both outputs. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SCAIL2ColoredMask", + display_name="Create SCAIL-2 Colored Mask", + category="model/conditioning/wan/scail", + inputs=[ + SAM3TrackData.Input("driving_track_data", tooltip="SAM3 track of the driving pose video. Will be rendered into the pose_video_mask output."), + io.MultiType.Input("ref_track_data", [SAM3TrackData, io.Mask], optional=True, display_name="reference_masks", + tooltip="SAM3 track of the reference image(s) (one identity per object, colored in batch order), or a plain MASK of the reference subject (rendered as a single identity)."), + io.String.Input("object_indices", default="", + tooltip="Comma-separated list of person indices to include (e.g. '0,2,3'). Applied to both reference and pose video masks. Empty = all."), + io.Combo.Input("sort_by", options=["none", "left_to_right", "area"], default="left_to_right", + tooltip="Order in which palette colors are assigned to the tracked objects (applied to both reference and pose video so each identity keeps the same color). Objects that appear in earlier frames always come first; within a frame, left_to_right = leftmost object (by centroid at first appearance) gets the first color, area = biggest object (by mask area at first appearance) gets the first color; none = keep SAM3's order."), + io.Boolean.Input("replacement_mode", default=False, + tooltip="False = Animation Mode (pose_video_mask has black background, reference_image_mask has white background). " + "True = Replacement Mode (pose_video_mask has white background, reference_image_mask has black background)."), + ], + outputs=[ + io.Image.Output("pose_video_mask"), + io.Image.Output("reference_image_mask"), + ], + is_experimental=True, + ) + + @classmethod + def execute(cls, driving_track_data, object_indices, sort_by, replacement_mode, ref_track_data=None): + def _prep(td): + masks_bool = _unpack(td) + if sort_by != "none" and masks_bool is not None: + first_t, cx, area = _first_appearance_cx_area(masks_bool) + if sort_by == "left_to_right": + order = sorted(range(len(cx)), key=lambda i: (first_t[i], cx[i])) + else: # "area" + order = sorted(range(len(area)), key=lambda i: (first_t[i], -area[i])) + td = _subset_track_data(td, order) + if object_indices.strip(): + indices = [int(i.strip()) for i in object_indices.split(",") if i.strip().isdigit()] + packed = td.get("packed_masks") + n_obj = packed.shape[1] if packed is not None else 0 + indices = [i for i in indices if 0 <= i < n_obj] + td = _subset_track_data(td, indices) + return td + + drv = _prep(driving_track_data) + # Animation: driving=black, ref=white. Replacement: driving=white, ref=black. + mask_video = _render_colored_masks(drv, "white" if replacement_mode else "black") + ref_bg = "black" if replacement_mode else "white" + + if ref_track_data is not None: + if isinstance(ref_track_data, torch.Tensor): # plain comfy MASK + reference_image_mask = _render_mask_as_identity(ref_track_data, ref_bg) + else: + reference_image_mask = _render_colored_masks(_prep(ref_track_data), ref_bg) + else: + H, W = drv["orig_size"] + fill_value = 1.0 if ref_bg == "white" else 0.0 + reference_image_mask = torch.full((1, H, W, 3), fill_value, device=comfy.model_management.intermediate_device(), dtype=comfy.model_management.intermediate_dtype()) + + return io.NodeOutput(mask_video, reference_image_mask) + + +class SCAILExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + WanSCAILToVideo, + SCAIL2ColoredMask, + ] + + +async def comfy_entrypoint() -> SCAILExtension: + return SCAILExtension() diff --git a/comfy_extras/nodes_sd3.py b/comfy_extras/nodes_sd3.py index 38cbf117b..40e84656b 100644 --- a/comfy_extras/nodes_sd3.py +++ b/comfy_extras/nodes_sd3.py @@ -13,8 +13,9 @@ class TripleCLIPLoader(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TripleCLIPLoader", - category="advanced/loaders", - description="[Recipes]\n\nsd3: clip-l, clip-g, t5", + display_name="Load CLIP (Triple)", + category="model/loaders", + description="Recipes:\nsd3: clip-l, clip-g, t5", inputs=[ io.Combo.Input("clip_name1", options=folder_paths.get_filename_list("text_encoders")), io.Combo.Input("clip_name2", options=folder_paths.get_filename_list("text_encoders")), @@ -41,7 +42,7 @@ class EmptySD3LatentImage(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptySD3LatentImage", - category="model/latent/sd3", + category="model/latent/stable diffusion", inputs=[ io.Int.Input("width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -66,7 +67,8 @@ class CLIPTextEncodeSD3(io.ComfyNode): return io.Schema( node_id="CLIPTextEncodeSD3", search_aliases=["sd3 prompt"], - category="advanced/conditioning", + display_name="CLIP Text Encode (SD3)", + category="model/conditioning/stable diffusion", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_sdpose.py b/comfy_extras/nodes_sdpose.py index 20d459b00..d1cbff2a6 100644 --- a/comfy_extras/nodes_sdpose.py +++ b/comfy_extras/nodes_sdpose.py @@ -96,8 +96,12 @@ class KeypointDraw: # Body connections - matching DWPose limbSeq (1-indexed, converted to 0-indexed) self.body_limbSeq = [ [2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], - [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], - [1, 16], [16, 18] + [10, 11], [2, 12], [12, 13], [13, 14] + ] + + # Head connections (1-indexed, converted to 0-indexed) + self.head_edges = [ + [2, 1], [1, 15], [15, 17], [1, 16], [16, 18] ] # Colors matching DWPose @@ -215,7 +219,7 @@ class KeypointDraw: return unique_pts if len(unique_pts) > 1 else [[center[0], center[1]], [center[0], center[1]]] def draw_wholebody_keypoints(self, canvas, keypoints, scores=None, threshold=0.3, - draw_body=True, draw_feet=True, draw_face=True, draw_hands=True, stick_width=4, face_point_size=3): + draw_body=True, draw_head=True, draw_feet=True, draw_face=True, draw_hands=True, stick_width=4, face_point_size=3): """ Draw wholebody keypoints (134 keypoints after processing) in DWPose style. @@ -237,9 +241,17 @@ class KeypointDraw: """ H, W, C = canvas.shape - # Draw body limbs - if draw_body and len(keypoints) >= 18: - for i, limb in enumerate(self.body_limbSeq): + # Draw body limbs & head connections + if (draw_body or draw_head) and len(keypoints) >= 18: + colorIndexOffset = 0 + edges = [] + if draw_body: + edges += self.body_limbSeq + else: + colorIndexOffset += len(self.body_limbSeq) + if draw_head: + edges += self.head_edges + for i, limb in enumerate(edges): # Convert from 1-indexed to 0-indexed idx1, idx2 = limb[0] - 1, limb[1] - 1 @@ -262,11 +274,17 @@ class KeypointDraw: polygon = self.draw.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stick_width), int(angle), 0, 360, 1) - self.draw.fillConvexPoly(canvas, polygon, self.colors[i % len(self.colors)]) + self.draw.fillConvexPoly(canvas, polygon, self.colors[(i + colorIndexOffset) % len(self.colors)]) - # Draw body keypoints - if draw_body and len(keypoints) >= 18: + # Draw body & head keypoints + if (draw_body or draw_head) and len(keypoints) >= 18: + head_keypoints = {0, 14, 15, 16, 17} # nose, eyes, ears + neck_point = 1 for i in range(18): + if not draw_head and i in head_keypoints: + continue + if not draw_body and i not in head_keypoints and i != neck_point: + continue if scores is not None and scores[i] < threshold: continue x, y = int(keypoints[i][0]), int(keypoints[i][1]) @@ -365,6 +383,7 @@ class SDPoseDrawKeypoints(io.ComfyNode): io.Int.Input("stick_width", default=4, min=1, max=10, step=1), io.Int.Input("face_point_size", default=3, min=1, max=10, step=1), io.Float.Input("score_threshold", default=0.3, min=0.0, max=1.0, step=0.01), + io.Boolean.Input("draw_head", default=True), ], outputs=[ io.Image.Output(), @@ -372,7 +391,7 @@ class SDPoseDrawKeypoints(io.ComfyNode): ) @classmethod - def execute(cls, keypoints, draw_body, draw_hands, draw_face, draw_feet, stick_width, face_point_size, score_threshold) -> io.NodeOutput: + def execute(cls, keypoints, draw_body, draw_hands, draw_face, draw_feet, stick_width, face_point_size, score_threshold, draw_head) -> io.NodeOutput: if not keypoints: return io.NodeOutput(torch.zeros((1, 64, 64, 3), dtype=torch.float32)) height = keypoints[0]["canvas_height"] @@ -405,7 +424,7 @@ class SDPoseDrawKeypoints(io.ComfyNode): canvas = drawer.draw_wholebody_keypoints( canvas, kp, sc, threshold=score_threshold, - draw_body=draw_body, draw_feet=draw_feet, + draw_body=draw_body, draw_head=draw_head, draw_feet=draw_feet, draw_face=draw_face, draw_hands=draw_hands, stick_width=stick_width, face_point_size=face_point_size, ) diff --git a/comfy_extras/nodes_sdupscale.py b/comfy_extras/nodes_sdupscale.py index ea283e971..5c247fb49 100644 --- a/comfy_extras/nodes_sdupscale.py +++ b/comfy_extras/nodes_sdupscale.py @@ -9,7 +9,7 @@ class SD_4XUpscale_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SD_4XUpscale_Conditioning", - category="model/conditioning/upscale_diffusion", + category="model/conditioning/stable diffusion upscaler", inputs=[ io.Image.Input("images"), io.Conditioning.Input("positive"), diff --git a/comfy_extras/nodes_seed.py b/comfy_extras/nodes_seed.py new file mode 100644 index 000000000..e64f1d7e3 --- /dev/null +++ b/comfy_extras/nodes_seed.py @@ -0,0 +1,33 @@ +import sys +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +class SeedNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedNode", + display_name="Seed", + search_aliases=["seed", "random"], + category="utilities", + inputs=[ + io.Int.Input("seed", min=0, max=sys.maxsize, control_after_generate=io.ControlAfterGenerate.fixed), + ], + outputs=[io.Int.Output(display_name="seed")], + ) + + @classmethod + def execute(cls, seed: int) -> io.NodeOutput: + return io.NodeOutput(seed) + + +class SeedExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [SeedNode] + + +async def comfy_entrypoint() -> SeedExtension: + return SeedExtension() diff --git a/comfy_extras/nodes_seedvr.py b/comfy_extras/nodes_seedvr.py new file mode 100644 index 000000000..c4ca3b55c --- /dev/null +++ b/comfy_extras/nodes_seedvr.py @@ -0,0 +1,614 @@ +import logging + +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +import torch + +import comfy.model_management +from comfy.ldm.seedvr.color_fix import ( + adain_color_transfer, + lab_color_transfer, + wavelet_color_transfer, +) +from comfy.ldm.seedvr.constants import ( + BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE, + SEEDVR2_ADAIN_SCALE_MULTIPLIER, + SEEDVR2_CHUNK_GIB_PER_MPX_FRAME, + SEEDVR2_CHUNK_RESERVED_GIB, + SEEDVR2_CHUNK_SIGMA_GIB, + SEEDVR2_CHUNK_SIGMA_K, + SEEDVR2_COLOR_MEM_HEADROOM, + SEEDVR2_DTYPE_BYTES_FLOOR, + SEEDVR2_LAB_SCALE_MULTIPLIER, + SEEDVR2_LATENT_CHANNELS, + SEEDVR2_OOM_BACKOFF_DIVISOR, + SEEDVR2_WAVELET_SCALE_MULTIPLIER, +) + +from torchvision.transforms import functional as TVF +from torchvision.transforms.functional import InterpolationMode + + +_SEEDVR2_INVALID_MODEL_MSG_PREFIX = "SeedVR2Conditioning: model object does not match expected SeedVR2 structure" +_ATTR_MISSING = object() + + +def _resolve_seedvr2_diffusion_model(model): + inner = getattr(model, "model", _ATTR_MISSING) + if inner is _ATTR_MISSING: + raise RuntimeError( + f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input has no 'model' attribute " + f"(got type {type(model).__name__})." + ) + if inner is None: + raise RuntimeError( + f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input.model is None " + f"(input type {type(model).__name__})." + ) + diffusion_model = getattr(inner, "diffusion_model", _ATTR_MISSING) + if diffusion_model is _ATTR_MISSING: + raise RuntimeError( + f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model' has no " + f"'diffusion_model' attribute (got type {type(inner).__name__})." + ) + if diffusion_model is None: + raise RuntimeError( + f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model.diffusion_model' " + f"is None (model.model type {type(inner).__name__})." + ) + return diffusion_model + + +def div_pad(image, factor): + height_factor, width_factor = factor + height, width = image.shape[-2:] + + pad_height = (height_factor - (height % height_factor)) % height_factor + pad_width = (width_factor - (width % width_factor)) % width_factor + + if pad_height == 0 and pad_width == 0: + return image + + padding = (0, pad_width, 0, pad_height) + return torch.nn.functional.pad(image, padding, mode='constant', value=0.0) + +def cut_videos(videos): + t = videos.size(1) + if t < 1: + raise ValueError("SeedVR2Preprocess expected at least one frame.") + if t == 1: + return videos + if t <= 4: + padding = videos[:, -1:].repeat(1, 4 - t + 1, 1, 1, 1) + return torch.cat([videos, padding], dim=1) + if (t - 1) % 4 == 0: + return videos + padding = videos[:, -1:].repeat(1, 4 - ((t - 1) % 4), 1, 1, 1) + videos = torch.cat([videos, padding], dim=1) + if (videos.size(1) - 1) % 4 != 0: + raise ValueError(f"SeedVR2Preprocess failed to pad video length to 4n+1; got {videos.size(1)} frames.") + return videos + +def _seedvr2_input_shorter_edge(images, node_name): + if images.dim() == 4: + return min(images.shape[1], images.shape[2]) + if images.dim() == 5: + return min(images.shape[2], images.shape[3]) + raise ValueError( + f"{node_name}: expected 4-D or 5-D IMAGE tensor, " + f"got shape {tuple(images.shape)}" + ) + + +def _seedvr2_pad(images, upscaled_shorter_edge, node_name): + if upscaled_shorter_edge < 2: + raise ValueError( + f"{node_name}: input shorter edge must be at least 2 pixels; " + f"got {upscaled_shorter_edge}." + ) + if images.shape[-1] > 3: + images = images[..., :3] + if images.dim() == 4: + # Comfy video components arrive as a 4-D IMAGE frame sequence: + # (frames, H, W, C). SeedVR2 consumes that as one video. + images = images.unsqueeze(0) + elif images.dim() != 5: + raise ValueError( + f"{node_name}: expected 4-D or 5-D IMAGE tensor, " + f"got shape {tuple(images.shape)}" + ) + images = images.permute(0, 1, 4, 2, 3) + + b, t, c, h, w = images.shape + images = images.reshape(b * t, c, h, w) + + images = torch.clamp(images, 0.0, 1.0) + images = div_pad(images, (16, 16)) + _, _, new_h, new_w = images.shape + + images = images.reshape(b, t, c, new_h, new_w) + images = cut_videos(images) + images_bthwc = images.permute(0, 1, 3, 4, 2).contiguous() + + return io.NodeOutput(images_bthwc) + + +class SeedVR2Preprocess(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedVR2Preprocess", + display_name="Pre-Process SeedVR2 Input", + category="image/pre-processors", + description="Pad a resized image for SeedVR2 model. Alpha channel is dropped. The node Post-Process SeedVR2 Output re-applies it from the original resized image.", + search_aliases=["seedvr2", "upscale", "video upscale", "pad", "preprocess"], + inputs=[ + io.Image.Input("resized_images", tooltip="The resized image to process."), + ], + outputs=[ + io.Image.Output("images", tooltip="The padded image for VAE encoding."), + ] + ) + + @classmethod + def execute(cls, resized_images): + upscaled_shorter_edge = _seedvr2_input_shorter_edge(resized_images, "SeedVR2Preprocess") + return _seedvr2_pad( + resized_images, upscaled_shorter_edge, "SeedVR2Preprocess", + ) + + +class SeedVR2PostProcessing(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedVR2PostProcessing", + display_name="Post-Process SeedVR2 Output", + category="image/post-processors", + description="Align the generated image with the original resized image and apply color correction.", + search_aliases=["seedvr2", "upscale", "color correction", "color match", "postprocess"], + inputs=[ + io.Image.Input("images", tooltip="The generated image to process."), + io.Image.Input("original_resized_images", tooltip="The original resized image before pre-processing, used as reference."), + io.Combo.Input("color_correction_method", options=["lab", "wavelet", "adain", "none"], default="lab", tooltip="Method to match the generated image colors to the original image. lab: transfer color in CIELAB space, preserving detail (most faithful). wavelet: transfer low-frequency color, keeping upscaled high-frequency detail. adain: match per-channel mean/std (fastest, global tint). none: skip color transfer (geometry alignment only)."), + ], + outputs=[io.Image.Output(display_name="images", tooltip="The aligned, color-corrected image.")], + ) + + @classmethod + def execute(cls, images, original_resized_images, color_correction_method): + alpha_input = None + if original_resized_images.shape[-1] == 4: + alpha_input = original_resized_images[..., 3:4] + original_resized_images = original_resized_images[..., :3] + decoded_5d, decoded_was_4d = cls._as_bthwc(images) + reference_full, _ = cls._as_bthwc(original_resized_images) + decoded_5d = cls._restore_reference_batch_time(decoded_5d, reference_full) + + b = min(decoded_5d.shape[0], reference_full.shape[0]) + t = min(decoded_5d.shape[1], reference_full.shape[1]) + reference_h = reference_full.shape[2] + reference_w = reference_full.shape[3] + + decoded_5d = decoded_5d[:b, :t, :, :, :] + target_h = min(decoded_5d.shape[2], reference_h) + target_w = min(decoded_5d.shape[3], reference_w) + decoded_5d = decoded_5d[:, :, :target_h, :target_w, :] + if color_correction_method in ("lab", "wavelet", "adain"): + reference_5d = reference_full[:b, :t, :, :, :] + reference_5d = cls._resize_reference(reference_5d, target_h, target_w) + output_device = decoded_5d.device + decoded_raw = cls._to_seedvr2_raw(decoded_5d) + reference_raw = cls._to_seedvr2_raw(reference_5d) + decoded_flat = decoded_raw.permute(0, 1, 4, 2, 3).reshape(b * t, decoded_raw.shape[4], target_h, target_w) + reference_flat = reference_raw.permute(0, 1, 4, 2, 3).reshape(b * t, reference_raw.shape[4], target_h, target_w) + output = cls._color_transfer_chunked( + decoded_flat, reference_flat, output_device, color_correction_method, + ) + output = output.reshape(b, t, output.shape[1], output.shape[2], output.shape[3]).permute(0, 1, 3, 4, 2) + output = output.add(1.0).div(2.0).clamp(0.0, 1.0) + elif color_correction_method == "none": + output = decoded_5d + else: + raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}") + + if alpha_input is not None: + alpha_5d, _ = cls._as_bthwc(alpha_input) + alpha_5d = alpha_5d[:output.shape[0], :output.shape[1], :output.shape[2], :output.shape[3], :] + output = torch.cat([output, alpha_5d.to(dtype=output.dtype, device=output.device)], dim=-1) + h2 = output.shape[-3] - (output.shape[-3] % 2) + w2 = output.shape[-2] - (output.shape[-2] % 2) + output = output[:, :, :h2, :w2, :] + if decoded_was_4d: + output = output.reshape(-1, output.shape[-3], output.shape[-2], output.shape[-1]) + return io.NodeOutput(output) + + @staticmethod + def _as_bthwc(images): + if images.ndim == 4: + return images.unsqueeze(0), True + if images.ndim == 5: + return images, False + raise ValueError( + f"SeedVR2PostProcessing: expected 4-D or 5-D IMAGE tensor, got shape {tuple(images.shape)}" + ) + + @staticmethod + def _restore_reference_batch_time(decoded, reference): + if decoded.shape[0] != 1: + return decoded + ref_b, ref_t = reference.shape[:2] + if ref_b < 1 or decoded.shape[1] % ref_b != 0: + return decoded + decoded_t = decoded.shape[1] // ref_b + if decoded_t < ref_t: + return decoded + return decoded.reshape(ref_b, decoded_t, decoded.shape[2], decoded.shape[3], decoded.shape[4]) + + @staticmethod + def _to_seedvr2_raw(images): + return images.mul(2.0).sub(1.0) + + @staticmethod + def _color_transfer_on_vae_device(decoded_flat, reference_flat, output_device, transfer_fn): + color_device = comfy.model_management.vae_device() + decoded_flat = decoded_flat.to(device=color_device) + reference_flat = reference_flat.to(device=color_device) + output = transfer_fn(decoded_flat, reference_flat) + return output.to(device=output_device) + + @staticmethod + def _lab_color_transfer_on_vae_device(decoded_flat, reference_flat, output_device): + color_device = comfy.model_management.vae_device() + result = None + for start in range(decoded_flat.shape[0]): + decoded_frame = decoded_flat[start:start + 1].to(device=color_device).clone() + reference_frame = reference_flat[start:start + 1].to(device=color_device).clone() + output = lab_color_transfer(decoded_frame, reference_frame).to(device=output_device) + if result is None: + result = torch.empty( + (decoded_flat.shape[0],) + tuple(output.shape[1:]), + device=output_device, + dtype=output.dtype, + ) + result[start:start + 1].copy_(output) + if result is None: + raise ValueError("SeedVR2PostProcessing: LAB color correction requires at least one frame.") + return result + + @classmethod + def _color_transfer_chunked(cls, decoded_flat, reference_flat, output_device, color_correction_method): + chunk_size = cls._estimate_color_correction_chunk_size(decoded_flat, color_correction_method) + while True: + try: + return cls._run_color_transfer_chunks( + decoded_flat, reference_flat, output_device, color_correction_method, chunk_size, + ) + except Exception as e: + comfy.model_management.raise_non_oom(e) + if chunk_size <= 1: + raise RuntimeError( + "SeedVR2PostProcessing: color correction OOM at one frame; " + f"color_correction_method={color_correction_method}, shape={tuple(decoded_flat.shape)}." + ) from e + chunk_size = max(1, chunk_size // SEEDVR2_OOM_BACKOFF_DIVISOR) + + @classmethod + def _run_color_transfer_chunks(cls, decoded_flat, reference_flat, output_device, color_correction_method, chunk_size): + result = None + for start in range(0, decoded_flat.shape[0], chunk_size): + end = min(start + chunk_size, decoded_flat.shape[0]) + decoded_chunk = decoded_flat[start:end] + reference_chunk = reference_flat[start:end] + if color_correction_method == "lab": + output = cls._lab_color_transfer_on_vae_device(decoded_chunk, reference_chunk, output_device) + elif color_correction_method == "wavelet": + output = cls._color_transfer_on_vae_device( + decoded_chunk, reference_chunk, output_device, wavelet_color_transfer, + ) + else: + output = cls._color_transfer_on_vae_device( + decoded_chunk, reference_chunk, output_device, adain_color_transfer, + ) + if result is None: + result = torch.empty( + (decoded_flat.shape[0],) + tuple(output.shape[1:]), + device=output_device, + dtype=output.dtype, + ) + result[start:end].copy_(output) + if result is None: + raise ValueError("SeedVR2PostProcessing: color correction requires at least one frame.") + return result + + @classmethod + def _estimate_color_correction_chunk_size(cls, decoded_flat, color_correction_method): + multiplier = cls._color_correction_memory_multiplier(color_correction_method) + frames = decoded_flat.shape[0] + _, channels, height, width = decoded_flat.shape + dtype_bytes = max(decoded_flat.element_size(), SEEDVR2_DTYPE_BYTES_FLOOR) + bytes_per_frame = height * width * channels * dtype_bytes * multiplier + if bytes_per_frame <= 0: + return frames + color_device = comfy.model_management.vae_device() + free_memory = comfy.model_management.get_free_memory(color_device) + chunk_size = int((free_memory * SEEDVR2_COLOR_MEM_HEADROOM) // bytes_per_frame) + return max(1, min(frames, chunk_size)) + + @staticmethod + def _color_correction_memory_multiplier(color_correction_method): + if color_correction_method == "lab": + return SEEDVR2_LAB_SCALE_MULTIPLIER + if color_correction_method == "wavelet": + return SEEDVR2_WAVELET_SCALE_MULTIPLIER + if color_correction_method == "adain": + return SEEDVR2_ADAIN_SCALE_MULTIPLIER + raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}") + + @staticmethod + def _resize_reference(reference, height, width): + if reference.shape[2] == height and reference.shape[3] == width: + return reference + b, t = reference.shape[:2] + reference_flat = reference.permute(0, 1, 4, 2, 3).reshape(b * t, reference.shape[4], reference.shape[2], reference.shape[3]) + resized = TVF.resize( + reference_flat, + size=(height, width), + interpolation=InterpolationMode.BICUBIC, + antialias=not (isinstance(reference_flat, torch.Tensor) and reference_flat.device.type == "mps"), + ) + return resized.reshape(b, t, resized.shape[1], height, width).permute(0, 1, 3, 4, 2) + + +class SeedVR2Conditioning(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedVR2Conditioning", + display_name="Apply SeedVR2 Conditioning", + category="model/conditioning", + description="Build SeedVR2 positive/negative conditioning from a VAE latent.", + search_aliases=["seedvr2", "upscale", "conditioning"], + inputs=[ + io.Model.Input("model", tooltip="The SeedVR2 model."), + io.Latent.Input("vae_conditioning", display_name="latent"), + ], + outputs=[ + io.Conditioning.Output(display_name="positive", tooltip="The positive conditioning for sampling."), + io.Conditioning.Output(display_name="negative", tooltip="The negative conditioning for sampling."), + ], + ) + + @classmethod + def execute(cls, model, vae_conditioning) -> io.NodeOutput: + + vae_conditioning = vae_conditioning["samples"] + if vae_conditioning.ndim != 5: + raise ValueError( + "SeedVR2Conditioning expects a 5-D VAE latent in Comfy " + f"channel-first layout; got shape {tuple(vae_conditioning.shape)}." + ) + if vae_conditioning.shape[1] != SEEDVR2_LATENT_CHANNELS: + if vae_conditioning.shape[-1] == SEEDVR2_LATENT_CHANNELS: + raise ValueError( + "SeedVR2Conditioning expects SeedVR2 VAE latents in Comfy " + f"channel-first layout (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); " + f"got channel-last shape {tuple(vae_conditioning.shape)}." + ) + raise ValueError( + "SeedVR2Conditioning expects SeedVR2 VAE latents with " + f"{SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(vae_conditioning.shape)}." + ) + vae_conditioning = vae_conditioning.movedim(1, -1).contiguous() + model = _resolve_seedvr2_diffusion_model(model) + pos_cond = model.positive_conditioning + neg_cond = model.negative_conditioning + + mask = vae_conditioning.new_ones(vae_conditioning.shape[:-1] + (1,)) + condition = torch.cat((vae_conditioning, mask), dim=-1) + condition = condition.movedim(-1, 1) + + negative = [[neg_cond.unsqueeze(0), {"condition": condition}]] + positive = [[pos_cond.unsqueeze(0), {"condition": condition}]] + + return io.NodeOutput(positive, negative) + +def _seedvr2_chunk_crossfade_weights(overlap, device, dtype): + """Descending previous-chunk weights across the overlap (next chunk gets ``1 - w``): a Hann fade over the middle third, flat shoulders on the outer thirds.""" + ramp = torch.linspace(0.0, 1.0, steps=overlap, device=device, dtype=dtype) + ramp = ((ramp - 1.0 / 3.0) / (1.0 / 3.0)).clamp(0.0, 1.0) + return 0.5 + 0.5 * torch.cos(torch.pi * ramp) + + +class SeedVR2TemporalChunk(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedVR2TemporalChunk", + display_name="Split SeedVR2 Latent", + category="model/latent/batch", + description="Split a SeedVR2 video latent into overlapping temporal chunks small enough to sample one at a time within VRAM, wiring latents outputs to both Apply SeedVR2 Conditioning and the sampler latent input before recombining with Merge SeedVR2 Latents.", + search_aliases=["seedvr2", "split", "chunk", "temporal", "video upscale", "rebatch"], + inputs=[ + io.Latent.Input("latent", tooltip="The VAE-encoded SeedVR2 latent to split."), + io.Int.Input("temporal_overlap", default=0, min=0, max=16384, + tooltip="Latent frames shared between adjacent chunks and crossfaded at merge; 0 = no overlap."), + io.DynamicCombo.Input("chunking_mode", + tooltip="manual = use frames_per_chunk exactly; auto = predict the largest chunk that fits free VRAM.", + options=[ + io.DynamicCombo.Option("auto", []), + io.DynamicCombo.Option("manual", [ + io.Int.Input("frames_per_chunk", default=21, min=1, max=16384, step=4, + tooltip="Pixel frames per temporal chunk (4n+1: 1, 5, 9, 13, ...)."), + ]), + ]), + ], + outputs=[ + io.Latent.Output(display_name="latents", is_output_list=True, + tooltip="The temporal chunks in sequence order."), + io.Int.Output(display_name="temporal_overlap", + tooltip="The effective latent-frame overlap between adjacent chunks, for Merge SeedVR2 Latents."), + ], + ) + + @classmethod + def execute(cls, latent, temporal_overlap, chunking_mode) -> io.NodeOutput: + samples = latent["samples"] + if samples.ndim != 5: + raise ValueError( + f"SeedVR2TemporalChunk: expected a 5-D video latent (B, C, T, H, W); " + f"got shape {tuple(samples.shape)}." + ) + if samples.shape[1] != SEEDVR2_LATENT_CHANNELS: + raise ValueError( + f"SeedVR2TemporalChunk: expected {SEEDVR2_LATENT_CHANNELS} latent channels; " + f"got shape {tuple(samples.shape)}." + ) + if temporal_overlap < 0: + raise ValueError( + f"SeedVR2TemporalChunk: temporal_overlap must be >= 0; got {temporal_overlap}." + ) + mode = chunking_mode["chunking_mode"] + if mode not in ("auto", "manual"): + raise ValueError( + f"SeedVR2TemporalChunk: chunking_mode must be 'auto' or 'manual'; " + f"got {mode!r}." + ) + t_latent = samples.shape[2] + t_pixel = 4 * (t_latent - 1) + 1 + + if mode == "auto": + free_gb = comfy.model_management.get_free_memory( + comfy.model_management.get_torch_device()) / (1024 ** 3) + mpx_per_frame = (samples.shape[0] * samples.shape[3] * samples.shape[4]) * (BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE ** 2) / 1e6 + budget_gb = free_gb - SEEDVR2_CHUNK_RESERVED_GIB - SEEDVR2_CHUNK_SIGMA_K * SEEDVR2_CHUNK_SIGMA_GIB + chunk_latent_max = max(1, int(budget_gb / (SEEDVR2_CHUNK_GIB_PER_MPX_FRAME * mpx_per_frame))) + frames_per_chunk = min(4 * (chunk_latent_max - 1) + 1, t_pixel) + logging.info( + "SeedVR2TemporalChunk auto: free=%.2fGiB, %.2fMpx -> frames_per_chunk=%d (t_pixel=%d).", + free_gb, mpx_per_frame, frames_per_chunk, t_pixel, + ) + else: + frames_per_chunk = chunking_mode["frames_per_chunk"] + if frames_per_chunk < 1 or (frames_per_chunk - 1) % 4 != 0: + raise ValueError( + f"SeedVR2TemporalChunk: frames_per_chunk must be a 4n+1 pixel-frame count " + f"(1, 5, 9, 13, 17, 21, ...); got {frames_per_chunk}." + ) + + if t_pixel <= frames_per_chunk: + return io.NodeOutput([latent], 0) + + chunk_latent = (frames_per_chunk - 1) // 4 + 1 + temporal_overlap = min(temporal_overlap, chunk_latent - 1) + step = chunk_latent - temporal_overlap + + chunks = [] + for start in range(0, t_latent, step): + end = min(start + chunk_latent, t_latent) + chunk = latent.copy() + chunk["samples"] = samples[:, :, start:end].contiguous() + chunks.append(chunk) + if end >= t_latent: + break + return io.NodeOutput(chunks, temporal_overlap) + + +class SeedVR2TemporalMerge(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedVR2TemporalMerge", + display_name="Merge SeedVR2 Latents", + category="model/latent/batch", + is_input_list=True, + description="Recombine sampled SeedVR2 latent temporal chunks into one latent, crossfading each overlap with a Hann window sized by the temporal_overlap wired from Split SeedVR2 Latent.", + search_aliases=["seedvr2", "merge", "temporal", "hann", "crossfade"], + inputs=[ + io.Latent.Input("latents", tooltip="The sampled temporal chunks in sequence order."), + io.Int.Input("temporal_overlap", default=0, min=0, max=16384, force_input=True, + tooltip="The temporal_overlap output of Split SeedVR2 Latent. 0 = plain concatenation."), + ], + outputs=[ + io.Latent.Output(display_name="latent", tooltip="The recombined full-length latent."), + ], + ) + + @classmethod + def execute(cls, latents, temporal_overlap) -> io.NodeOutput: + temporal_overlap = temporal_overlap[0] + if temporal_overlap < 0: + raise ValueError( + f"SeedVR2TemporalMerge: temporal_overlap must be >= 0; got {temporal_overlap}." + ) + chunks = [entry["samples"] for entry in latents] + first = chunks[0] + if first.ndim != 5: + raise ValueError( + f"SeedVR2TemporalMerge: expected 5-D video latents (B, C, T, H, W); " + f"chunk 0 has shape {tuple(first.shape)}." + ) + for i, chunk in enumerate(chunks[1:], start=1): + if chunk.shape[:2] != first.shape[:2] or chunk.shape[3:] != first.shape[3:]: + raise ValueError( + f"SeedVR2TemporalMerge: chunk {i} shape {tuple(chunk.shape)} does not " + f"match chunk 0 shape {tuple(first.shape)} outside the temporal axis." + ) + if i < len(chunks) - 1 and chunk.shape[2] != first.shape[2]: + raise ValueError( + f"SeedVR2TemporalMerge: chunk {i} has {chunk.shape[2]} latent frames but " + f"chunk 0 has {first.shape[2]}; only the final chunk may be shorter." + ) + + out = latents[0].copy() + out.pop("noise_mask", None) + + if len(chunks) == 1: + out["samples"] = first + return io.NodeOutput(out) + if temporal_overlap == 0: + out["samples"] = torch.cat(chunks, dim=2) + return io.NodeOutput(out) + + chunk_latent = first.shape[2] + step = chunk_latent - min(temporal_overlap, chunk_latent - 1) + t_total = step * (len(chunks) - 1) + chunks[-1].shape[2] + b, c, _, h, w = first.shape + merged = torch.empty((b, c, t_total, h, w), device=first.device, dtype=first.dtype) + + merged[:, :, :chunk_latent] = first + filled = chunk_latent + for i, chunk in enumerate(chunks[1:], start=1): + start = i * step + end = start + chunk.shape[2] + # Crossfade width is bounded by the previous fill frontier and by a runt + # final chunk shorter than the configured overlap. + fade = min(filled - start, chunk.shape[2]) + if fade > 0: + w_prev = _seedvr2_chunk_crossfade_weights( + fade, chunk.device, chunk.dtype).view(1, 1, fade, 1, 1) + merged[:, :, start:start + fade] = ( + merged[:, :, start:start + fade] * w_prev + chunk[:, :, :fade] * (1.0 - w_prev) + ) + merged[:, :, start + fade:end] = chunk[:, :, fade:] + else: + merged[:, :, start:end] = chunk + filled = end + + out["samples"] = merged + return io.NodeOutput(out) + + +class SeedVRExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + SeedVR2Conditioning, + SeedVR2Preprocess, + SeedVR2PostProcessing, + SeedVR2TemporalChunk, + SeedVR2TemporalMerge, + ] + +async def comfy_entrypoint() -> SeedVRExtension: + return SeedVRExtension() diff --git a/comfy_extras/nodes_stable3d.py b/comfy_extras/nodes_stable3d.py index 8a6e5b726..b0eba819b 100644 --- a/comfy_extras/nodes_stable3d.py +++ b/comfy_extras/nodes_stable3d.py @@ -27,7 +27,7 @@ class StableZero123_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableZero123_Conditioning", - category="model/conditioning/3d_models", + category="model/conditioning/stable zero123", inputs=[ io.ClipVision.Input("clip_vision"), io.Image.Input("init_image"), @@ -65,7 +65,7 @@ class StableZero123_Conditioning_Batched(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableZero123_Conditioning_Batched", - category="model/conditioning/3d_models", + category="model/conditioning/stable zero123", inputs=[ io.ClipVision.Input("clip_vision"), io.Image.Input("init_image"), @@ -112,7 +112,7 @@ class SV3D_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SV3D_Conditioning", - category="model/conditioning/3d_models", + category="model/conditioning/stable video 3d", inputs=[ io.ClipVision.Input("clip_vision"), io.Image.Input("init_image"), diff --git a/comfy_extras/nodes_stable_cascade.py b/comfy_extras/nodes_stable_cascade.py index e55f248ae..ddfb4f2b0 100644 --- a/comfy_extras/nodes_stable_cascade.py +++ b/comfy_extras/nodes_stable_cascade.py @@ -29,7 +29,7 @@ class StableCascade_EmptyLatentImage(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_EmptyLatentImage", - category="model/latent/stable_cascade", + category="model/latent/stable cascade", inputs=[ io.Int.Input("width", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8), io.Int.Input("height", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8), @@ -58,7 +58,7 @@ class StableCascade_StageC_VAEEncode(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_StageC_VAEEncode", - category="model/latent/stable_cascade", + category="model/latent/stable cascade", inputs=[ io.Image.Input("image"), io.Vae.Input("vae"), @@ -93,7 +93,7 @@ class StableCascade_StageB_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_StageB_Conditioning", - category="model/conditioning/stable_cascade", + category="model/conditioning/stable cascade", inputs=[ io.Conditioning.Input("conditioning"), io.Latent.Input("stage_c"), @@ -119,7 +119,7 @@ class StableCascade_SuperResolutionControlnet(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_SuperResolutionControlnet", - category="experimental/stable_cascade", + category="experimental/stable cascade", is_experimental=True, inputs=[ io.Image.Input("image"), diff --git a/comfy_extras/nodes_string.py b/comfy_extras/nodes_string.py index 97485c8c5..21929ae63 100644 --- a/comfy_extras/nodes_string.py +++ b/comfy_extras/nodes_string.py @@ -440,6 +440,57 @@ class JsonExtractString(io.ComfyNode): except (json.JSONDecodeError, TypeError): return io.NodeOutput("") + +def _dump_json(value, indent): + return json.dumps(value, ensure_ascii=False, indent=indent or None) + + +class ConvertDictionaryToString(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ConvertDictionaryToString", + display_name="Convert Dictionary to String", + category="text", + search_aliases=["json", "dict to json", "stringify", "serialize", "dict to string"], + inputs=[ + io.Dict.Input("dictionary"), + io.Int.Input("indent", default=2, min=0, max=8, + tooltip="Spaces per indent level. 0 produces compact single-line string."), + ], + outputs=[ + io.String.Output(), + ], + ) + + @classmethod + def execute(cls, dictionary, indent=2): + return io.NodeOutput(_dump_json(dictionary, indent)) + + +class ConvertArrayToString(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ConvertArrayToString", + display_name="Convert Array to String", + category="text", + search_aliases=["json", "list to json", "stringify", "serialize", "list to string", "array to json"], + inputs=[ + io.Array.Input("array"), + io.Int.Input("indent", default=2, min=0, max=8, + tooltip="Spaces per indent level. 0 produces compact single-line string."), + ], + outputs=[ + io.String.Output(), + ], + ) + + @classmethod + def execute(cls, array, indent=2): + return io.NodeOutput(_dump_json(array, indent)) + + class StringExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: @@ -457,6 +508,8 @@ class StringExtension(ComfyExtension): RegexExtract, RegexReplace, JsonExtractString, + ConvertDictionaryToString, + ConvertArrayToString, ] async def comfy_entrypoint() -> StringExtension: diff --git a/comfy_extras/nodes_text.py b/comfy_extras/nodes_text.py new file mode 100644 index 000000000..a485f5df8 --- /dev/null +++ b/comfy_extras/nodes_text.py @@ -0,0 +1,71 @@ +import os +import json +from typing_extensions import override +from comfy_api.latest import io, ComfyExtension, ui +import folder_paths + + +class SaveTextNode(io.ComfyNode): + """Save text content to .txt, .md, or .json.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SaveText", + search_aliases=["save text", "write text", "export text"], + display_name="Save Text", + category="text", + description="Save text content to a file in the output directory.", + inputs=[ + io.String.Input("text", force_input=True), + io.String.Input("filename_prefix", default="ComfyUI"), + io.Combo.Input("format", options=["txt", "md", "json"], default="txt"), + ], + outputs=[io.String.Output(display_name="text")], + is_output_node=True, + ) + + @classmethod + def execute(cls, text, filename_prefix, format): + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path( + filename_prefix, + folder_paths.get_output_directory(), + 1, + 1, + ) + + file = f"{filename}_{counter:05}.{format}" + filepath = os.path.join(full_output_folder, file) + + if format == "json": + # tries to pretty print otherwise saves normally + try: + data = json.loads(text) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + except json.JSONDecodeError: + with open(filepath, "w", encoding="utf-8") as f: + f.write(text) + else: + with open(filepath, "w", encoding="utf-8") as f: + f.write(text) + + return io.NodeOutput( + text, + ui={ + "text": (text,), + "files": [ + ui.SavedResult(file, subfolder, io.FolderType.output) + ] + } + ) + +class TextExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + SaveTextNode + ] + +async def comfy_entrypoint() -> TextExtension: + return TextExtension() diff --git a/comfy_extras/nodes_text_overlay.py b/comfy_extras/nodes_text_overlay.py new file mode 100644 index 000000000..4c5cdae60 --- /dev/null +++ b/comfy_extras/nodes_text_overlay.py @@ -0,0 +1,150 @@ +import numpy as np +import torch +from PIL import Image as PILImage, ImageColor, ImageDraw, ImageFont +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, IO + + +class TextOverlay(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TextOverlay", + display_name="Draw Text Overlay", + category="text", + description="Draw text overlay on an image or batch of images.", + search_aliases=["text", "label", "caption", "subtitle", "watermark", "title", "addlabel", "overlay"], + inputs=[ + IO.Image.Input("images"), + IO.String.Input("text", multiline=True, default=""), + IO.Float.Input("font_size", default=5.0, min=0.5, max=50.0, step=0.5, tooltip="Font size as a percentage of the image height."), + IO.Color.Input("color", default="#ffffff", tooltip="Color of the text."), + IO.Combo.Input("position", options=["top", "bottom"], default="top"), + IO.Combo.Input("align", options=["left", "center", "right"], default="left"), + IO.Boolean.Input("outline", default=True, tooltip="Draw a black outline around the text."), + ], + outputs=[IO.Image.Output(display_name="images")], + ) + + @classmethod + def execute(cls, images, text, font_size, color, position, align, outline) -> IO.NodeOutput: + if text.strip() == "": + return IO.NodeOutput(images) + + text = text.replace("\\n", "\n").replace("\\t", "\t") + + text_rgba = cls.parse_color_to_rgba(color) + outline_rgba = (0, 0, 0, 255) if outline else (0, 0, 0, 0) + + # Render the overlay once and composite it across all frames in the batch + height = images.shape[1] + width = images.shape[2] + overlay_rgb, overlay_alpha = cls.render_overlay_text(width, height, text, position, align, font_size, text_rgba, outline_rgba) + overlay_rgb = overlay_rgb.to(device=images.device, dtype=images.dtype) + overlay_alpha = overlay_alpha.to(device=images.device, dtype=images.dtype) + + result = images * (1.0 - overlay_alpha) + overlay_rgb * overlay_alpha + return IO.NodeOutput(result) + + @staticmethod + def parse_color_to_rgba(color_string): + parsed = ImageColor.getrgb(color_string) + + if len(parsed) == 3: + return (*parsed, 255) + + return parsed + + @classmethod + def render_overlay_text(cls, width, height, text, position, align, font_size, text_rgba, outline_rgba): + line_spacing = 1.2 + margin_percent = 1.0 + min_font_percent = 2.0 + min_font_pixels = 10 + outline_thickness_factor = 0.04 + + # Draw onto a transparent layer so the result can be alpha-composited over any frame. + layer = PILImage.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + + margin = int(round(margin_percent / 100.0 * min(width, height))) + max_width = max(1, width - 2 * margin) + max_height = max(1, height - 2 * margin) + + # Font scales with resolution, then shrinks to fit the height. + size = max(1, int(round(font_size / 100.0 * height))) + floor = min(size, max(min_font_pixels, int(round(min_font_percent / 100.0 * height)))) + + while True: + font = ImageFont.load_default(size=size) + stroke = max(1, int(round(size * outline_thickness_factor))) if outline_rgba[3] > 0 else 0 + block = "\n".join(cls.wrap_text(text, font, max_width)) + # convert line spacing to pixel spacing + single = draw.textbbox((0, 0), "Ay", font=font, stroke_width=stroke) + double = draw.multiline_textbbox((0, 0), "Ay\nAy", font=font, spacing=0, stroke_width=stroke) + natural_advance = (double[3] - double[1]) - (single[3] - single[1]) + pixel_spacing = int(round(size * line_spacing - natural_advance)) + box = draw.multiline_textbbox((0, 0), block, font=font, spacing=pixel_spacing, stroke_width=stroke) + block_height = box[3] - box[1] + + if block_height <= max_height or size <= floor: + break + + size = max(floor, int(size * 0.9)) + + anchor_h, x = {"left": ("l", margin), "center": ("m", width / 2), "right": ("r", width - margin)}[align] + + # Offset y so the rendered text sits flush against the margin + if position == "bottom": + y = height - margin - box[3] + else: + y = margin - box[1] + + draw.multiline_text((x, y), block, font=font, fill=text_rgba, anchor=anchor_h + "a", + align=align, spacing=pixel_spacing, stroke_width=stroke, stroke_fill=outline_rgba) + + overlay = np.array(layer).astype(np.float32) / 255.0 + overlay_rgb = torch.from_numpy(overlay[:, :, :3]) + overlay_alpha = torch.from_numpy(overlay[:, :, 3:4]) + return overlay_rgb, overlay_alpha + + @staticmethod + def wrap_text(text, font, max_width): + lines = [] + for raw_line in text.split("\n"): + words = raw_line.split() + if not words: + lines.append("") + continue + current = "" + # Break the line into words and split words that are too long + for word in words: + while font.getlength(word) > max_width and len(word) > 1: + cut = 1 + while cut < len(word) and font.getlength(word[:cut + 1]) <= max_width: + cut += 1 + if current: + lines.append(current) + current = "" + lines.append(word[:cut]) + word = word[cut:] + candidate = word if not current else current + " " + word + if not current or font.getlength(candidate) <= max_width: + current = candidate + else: + lines.append(current) + current = word + if current: + lines.append(current) + return lines + + +class TextOverlayExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [TextOverlay] + + +async def comfy_entrypoint() -> TextOverlayExtension: + return TextOverlayExtension() diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index d52faf815..5a947d5c5 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -35,7 +35,7 @@ class TextGenerate(io.ComfyNode): io.Image.Input("image", optional=True), io.Image.Input("video", optional=True, tooltip="Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."), io.Audio.Input("audio", optional=True), - io.Int.Input("max_length", default=256, min=1, max=2048), + io.Int.Input("max_length", default=512, min=1, max=32768), io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"), io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."), io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True), diff --git a/comfy_extras/nodes_train.py b/comfy_extras/nodes_train.py index 046eeaaf5..a27217b80 100644 --- a/comfy_extras/nodes_train.py +++ b/comfy_extras/nodes_train.py @@ -15,6 +15,7 @@ import comfy.sampler_helpers import comfy.sd import comfy.utils import comfy.model_management +from comfy.conds import CONDRegular, CONDList from comfy.cli_args import args, PerformanceFeature import comfy_extras.nodes_custom_sampler import folder_paths @@ -120,6 +121,11 @@ def process_cond_list(d, prefix=""): process_cond_list(v, f"{prefix}.{k}") elif isinstance(v, torch.Tensor): d[k] = v.clone() + elif isinstance(v, CONDList): + v.cond = [t.detach() if isinstance(t, torch.Tensor) else t for t in v.cond] + elif isinstance(v, CONDRegular): + if isinstance(v.cond, torch.Tensor): + v.cond = v.cond.detach() elif isinstance(v, (list, tuple)): for index, item in enumerate(v): process_cond_list(item, f"{prefix}.{k}.{index}") @@ -1143,45 +1149,45 @@ class TrainLoraNode(io.ComfyNode): # Process conditioning positive = _process_conditioning(positive) - # Setup model and dtype - mp = model.clone() - use_grad_scaler = False - lora_dtype = node_helpers.string_to_torch_dtype(lora_dtype) - if training_dtype != "none": - dtype = node_helpers.string_to_torch_dtype(training_dtype) - mp.set_model_compute_dtype(dtype) - else: - # Detect model's native dtype for autocast - model_dtype = mp.model.get_dtype() - if model_dtype == torch.float16: - dtype = torch.float16 - # GradScaler only supports float16 gradients, not bfloat16. - # Only enable it when lora params will also be in float16. - if lora_dtype != torch.bfloat16: - use_grad_scaler = True - # Warn about fp16 accumulation instability during training - if PerformanceFeature.Fp16Accumulation in args.fast: - logging.warning( - "WARNING: FP16 model detected with fp16_accumulation enabled. " - "This combination can be numerically unstable during training and may cause NaN values. " - "Suggested fixes: 1) Set training_dtype to 'bf16', or 2) Disable fp16_accumulation (remove from --fast flags)." - ) - else: - # For fp8, bf16, or other dtypes, use bf16 autocast - dtype = torch.bfloat16 - - # Prepare latents and compute counts - latents_dtype = dtype if dtype not in (None,) else torch.bfloat16 - latents, num_images, multi_res = _prepare_latents_and_count( - latents, latents_dtype, bucket_mode - ) - - # Validate and expand conditioning - positive = _validate_and_expand_conditioning(positive, num_images, bucket_mode) - with torch.inference_mode(False): + # Setup model and dtype + mp = model.clone(force_deepcopy=True) + use_grad_scaler = False + lora_dtype = node_helpers.string_to_torch_dtype(lora_dtype) + if training_dtype != "none": + dtype = node_helpers.string_to_torch_dtype(training_dtype) + mp.set_model_compute_dtype(dtype) + else: + # Detect model's native dtype for autocast + model_dtype = mp.model.get_dtype() + if model_dtype == torch.float16: + dtype = torch.float16 + # GradScaler only supports float16 gradients, not bfloat16. + # Only enable it when lora params will also be in float16. + if lora_dtype != torch.bfloat16: + use_grad_scaler = True + # Warn about fp16 accumulation instability during training + if PerformanceFeature.Fp16Accumulation in args.fast: + logging.warning( + "WARNING: FP16 model detected with fp16_accumulation enabled. " + "This combination can be numerically unstable during training and may cause NaN values. " + "Suggested fixes: 1) Set training_dtype to 'bf16', or 2) Disable fp16_accumulation (remove from --fast flags)." + ) + else: + # For fp8, bf16, or other dtypes, use bf16 autocast + dtype = torch.bfloat16 + + # Prepare latents and compute counts + latents_dtype = dtype if dtype not in (None,) else torch.bfloat16 + latents, num_images, multi_res = _prepare_latents_and_count( + latents, latents_dtype, bucket_mode + ) + + # Validate and expand conditioning + positive = _validate_and_expand_conditioning(positive, num_images, bucket_mode) + # Setup models for training - mp.model.requires_grad_(False) + mp.model.requires_grad_(False).train() # Load existing LoRA weights if provided existing_weights, existing_steps = _load_existing_lora(existing_lora) @@ -1361,7 +1367,7 @@ class SaveLoRA(io.ComfyNode): node_id="SaveLoRA", search_aliases=["export lora"], display_name="Save LoRA Weights", - category="advanced/model_merging", + category="model/merging", is_experimental=True, is_output_node=True, inputs=[ diff --git a/comfy_extras/nodes_triposplat.py b/comfy_extras/nodes_triposplat.py index 1848ad31a..c892213e4 100644 --- a/comfy_extras/nodes_triposplat.py +++ b/comfy_extras/nodes_triposplat.py @@ -65,7 +65,7 @@ class TripoSplatPreprocessImage(IO.ComfyNode): return IO.Schema( node_id="TripoSplatPreprocessImage", display_name="TripoSplat Preprocess Image", - category="3d/conditioning", + category="model/conditioning/triposplat", description="Crop center each image to a square canvas on a black background and add padding.", inputs=[ IO.Image.Input("image"), @@ -95,7 +95,7 @@ class TripoSplatConditioning(IO.ComfyNode): return IO.Schema( node_id="TripoSplatConditioning", display_name="TripoSplat Conditioning", - category="3d/conditioning", + category="model/conditioning/triposplat", description="Encode the image with DINOv3 and the Flux2 VAE into TripoSplat positive/negative " "conditioning, and create the fixed size noise target (latent + camera) for the KSampler", inputs=[ @@ -143,7 +143,7 @@ class VAEDecodeTripoSplat(IO.ComfyNode): return IO.Schema( node_id="VAEDecodeTripoSplat", display_name="TripoSplat Decode", - category="3d/latent", + category="model/latent/triposplat", description="Decode the sampled TripoSplat latent into a 3D gaussian splat. " "Modify the number of gaussians to vary the density.", inputs=[ @@ -188,7 +188,7 @@ class TripoSplatSamplingPreview(IO.ComfyNode): return IO.Schema( node_id="TripoSplatSamplingPreview", display_name="TripoSplat Sampling Preview", - category="3d/latent", + category="model/latent/triposplat", description="Patch the TripoSplat model for the standard Ksampler node to show a live decoded " "gaussian splat preview at each step.", inputs=[ diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.py index ae1d826d5..3bfd00be4 100644 --- a/comfy_extras/nodes_video.py +++ b/comfy_extras/nodes_video.py @@ -19,7 +19,7 @@ class SaveWEBM(io.ComfyNode): category="video", is_experimental=True, inputs=[ - io.Image.Input("images"), + io.Image.Input("images", tooltip="RGBA images are saved with their alpha channel as transparency (vp9 codec only)."), io.String.Input("filename_prefix", default="ComfyUI"), io.Combo.Input("codec", options=["vp9", "av1"]), io.Float.Input("fps", default=24.0, min=0.01, max=1000.0, step=0.01), @@ -27,6 +27,7 @@ class SaveWEBM(io.ComfyNode): ], hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo], is_output_node=True, + outputs=[io.Image.Output(display_name="images")] ) @classmethod @@ -45,24 +46,31 @@ class SaveWEBM(io.ComfyNode): for x in cls.hidden.extra_pnginfo: container.metadata[x] = json.dumps(cls.hidden.extra_pnginfo[x]) + # Save transparency when the images carry an alpha channel (RGBA) and the codec supports it. + # vp9 -> yuva420p; other codecs have no usable alpha path, so the alpha is ignored. + save_alpha = images.shape[-1] == 4 and codec == "vp9" + codec_map = {"vp9": "libvpx-vp9", "av1": "libsvtav1"} stream = container.add_stream(codec_map[codec], rate=Fraction(round(fps * 1000), 1000)) stream.width = images.shape[-2] stream.height = images.shape[-3] - stream.pix_fmt = "yuv420p10le" if codec == "av1" else "yuv420p" + stream.pix_fmt = "yuva420p" if save_alpha else ("yuv420p10le" if codec == "av1" else "yuv420p") stream.bit_rate = 0 stream.options = {'crf': str(crf)} if codec == "av1": stream.options["preset"] = "6" for frame in images: - frame = av.VideoFrame.from_ndarray(torch.clamp(frame[..., :3] * 255, min=0, max=255).to(device=torch.device("cpu"), dtype=torch.uint8).numpy(), format="rgb24") + if save_alpha: + frame = av.VideoFrame.from_ndarray(torch.clamp(frame[..., :4] * 255, min=0, max=255).to(device=torch.device("cpu"), dtype=torch.uint8).numpy(), format="rgba") + else: + frame = av.VideoFrame.from_ndarray(torch.clamp(frame[..., :3] * 255, min=0, max=255).to(device=torch.device("cpu"), dtype=torch.uint8).numpy(), format="rgb24") for packet in stream.encode(frame): container.mux(packet) container.mux(stream.encode()) container.close() - return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) + return io.NodeOutput(images, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) class SaveVideo(io.ComfyNode): @classmethod @@ -73,7 +81,7 @@ class SaveVideo(io.ComfyNode): display_name="Save Video", category="video", essentials_category="Basics", - description="Saves the input images to your ComfyUI output directory.", + description="Saves the input videos to your ComfyUI output directory.", inputs=[ io.Video.Input("video", tooltip="The video to save."), io.String.Input("filename_prefix", default="video/ComfyUI", tooltip="The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."), @@ -82,6 +90,7 @@ class SaveVideo(io.ComfyNode): ], hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo], is_output_node=True, + outputs=[io.Video.Output("video")], ) @classmethod @@ -110,7 +119,7 @@ class SaveVideo(io.ComfyNode): metadata=saved_metadata ) - return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) + return io.NodeOutput(video, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) class CreateVideo(io.ComfyNode): @@ -127,6 +136,17 @@ class CreateVideo(io.ComfyNode): io.Image.Input("images", tooltip="The images to create a video from."), io.Float.Input("fps", default=30.0, min=1.0, max=120.0, step=1.0), io.Audio.Input("audio", optional=True, tooltip="The audio to add to the video."), + io.Int.Input( + "bit_depth", + min=8, + max=10, + default=8, + step=2, + tooltip="Bit depth of the created video. 10-bit keeps smoother gradients with less" + " banding, but some players and downstream nodes may not support it.", + optional=True, + display_mode=io.NumberDisplay.number, + ), ], outputs=[ io.Video.Output(), @@ -134,9 +154,14 @@ class CreateVideo(io.ComfyNode): ) @classmethod - def execute(cls, images: Input.Image, fps: float, audio: Optional[Input.Audio] = None) -> io.NodeOutput: + def execute( + cls, images: Input.Image, fps: float, audio: Optional[Input.Audio] = None, bit_depth: int = 8, + ) -> io.NodeOutput: return io.NodeOutput( - InputImpl.VideoFromComponents(Types.VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps))) + InputImpl.VideoFromComponents( + Types.VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps)), + bit_depth=bit_depth, + ) ) class GetVideoComponents(io.ComfyNode): @@ -147,7 +172,7 @@ class GetVideoComponents(io.ComfyNode): search_aliases=["extract frames", "split video", "video to images", "demux"], display_name="Get Video Components", category="video", - description="Extracts all components from a video: frames, audio, and framerate.", + description="Extracts all components from a video: frames, audio, framerate, and bit depth.", inputs=[ io.Video.Input("video", tooltip="The video to extract components from."), ], @@ -155,13 +180,14 @@ class GetVideoComponents(io.ComfyNode): io.Image.Output(display_name="images"), io.Audio.Output(display_name="audio"), io.Float.Output(display_name="fps"), + io.Int.Output(display_name="bit_depth"), ], ) @classmethod def execute(cls, video: Input.Video) -> io.NodeOutput: components = video.get_components() - return io.NodeOutput(components.images, components.audio, float(components.frame_rate)) + return io.NodeOutput(components.images, components.audio, float(components.frame_rate), video.get_bit_depth()) class LoadVideo(io.ComfyNode): @@ -209,13 +235,8 @@ class VideoSlice(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Video Slice", - display_name="Video Slice", - search_aliases=[ - "trim video duration", - "skip first frames", - "frame load cap", - "start time", - ], + display_name="Trim Video", + search_aliases=["trim video duration", "skip first frames", "frame load cap", "start time"], category="video", essentials_category="Video Tools", inputs=[ diff --git a/comfy_extras/nodes_video_model.py b/comfy_extras/nodes_video_model.py index 0d6cae6a8..01d48d4d4 100644 --- a/comfy_extras/nodes_video_model.py +++ b/comfy_extras/nodes_video_model.py @@ -41,7 +41,7 @@ class SVD_img2vid_Conditioning: FUNCTION = "encode" - CATEGORY = "model/conditioning/video_models" + CATEGORY = "model/conditioning/stable video" def encode(self, clip_vision, init_image, vae, width, height, video_frames, motion_bucket_id, fps, augmentation_level): output = clip_vision.encode_image(init_image) @@ -108,7 +108,7 @@ class VideoTriangleCFGGuidance: return (m, ) class ImageOnlyCheckpointSave(comfy_extras.nodes_model_merging.CheckpointSave): - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" @classmethod def INPUT_TYPES(s): @@ -138,7 +138,7 @@ class ConditioningSetAreaPercentageVideo: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, width, height, temporal, x, y, z, strength): c = node_helpers.conditioning_set_values(conditioning, {"area": ("percentage", temporal, height, width, z, y, x), @@ -160,4 +160,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ImageOnlyCheckpointLoader": "Load Checkpoint Image Only (img2vid model)", "VideoLinearCFGGuidance": "Video Linear CFG Guidance", "VideoTriangleCFGGuidance": "Video Triangle CFG Guidance", + "ConditioningSetAreaPercentageVideo": "Conditioning (Set Area with Percentage for Video)", } diff --git a/comfy_extras/nodes_void.py b/comfy_extras/nodes_void.py index b43154b8d..7527baf43 100644 --- a/comfy_extras/nodes_void.py +++ b/comfy_extras/nodes_void.py @@ -175,7 +175,7 @@ class VOIDInpaintConditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="VOIDInpaintConditioning", - category="model/conditioning/video_models", + category="model/conditioning/void", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -288,7 +288,7 @@ class VOIDWarpedNoise(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="VOIDWarpedNoise", - category="model/latent/video", + category="model/latent/void", inputs=[ OpticalFlow.Input( "optical_flow", @@ -393,7 +393,7 @@ class VOIDWarpedNoiseSource(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="VOIDWarpedNoiseSource", - category="model/sampling/noise", + category="model/latent/void", inputs=[ io.Latent.Input("warped_noise", tooltip="Warped noise latent from VOIDWarpedNoise"), diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.py index 67d3a8443..0e47a58df 100644 --- a/comfy_extras/nodes_wan.py +++ b/comfy_extras/nodes_wan.py @@ -18,7 +18,7 @@ class WanImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -66,7 +66,7 @@ class WanFunControlToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanFunControlToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/fun control", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -119,7 +119,7 @@ class Wan22FunControlToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Wan22FunControlToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/fun control", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -184,7 +184,7 @@ class WanFirstLastFrameToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanFirstLastFrameToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -256,7 +256,7 @@ class WanFunInpaintToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanFunInpaintToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/fun inpaint", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -288,7 +288,7 @@ class WanVaceToVideo(io.ComfyNode): return io.Schema( node_id="WanVaceToVideo", search_aliases=["video conditioning", "video control"], - category="model/conditioning/video_models", + category="model/conditioning/wan/vace", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -375,7 +375,8 @@ class TrimVideoLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TrimVideoLatent", - category="model/latent/video", + display_name="Trim Video Latent", + category="model/latent", inputs=[ io.Latent.Input("samples"), io.Int.Input("trim_amount", default=0, min=0, max=99999), @@ -398,7 +399,7 @@ class WanCameraImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanCameraImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/camera", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -452,7 +453,7 @@ class WanPhantomSubjectToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanPhantomSubjectToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/phantom subject", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -707,7 +708,7 @@ class WanTrackToVideo(io.ComfyNode): return io.Schema( node_id="WanTrackToVideo", search_aliases=["motion tracking", "trajectory video", "point tracking", "keypoint animation"], - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -951,7 +952,7 @@ class WanSoundImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanSoundImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/sound", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -984,7 +985,7 @@ class WanSoundImageToVideoExtend(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanSoundImageToVideoExtend", - category="model/conditioning/video_models", + category="model/conditioning/wan/sound", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -1046,7 +1047,7 @@ class WanHuMoImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanHuMoImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/humo", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -1112,7 +1113,7 @@ class WanAnimateToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanAnimateToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/animate", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -1252,7 +1253,7 @@ class Wan22ImageToVideoLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Wan22ImageToVideoLatent", - category="model/conditioning/inpaint", + category="model/conditioning/wan", inputs=[ io.Vae.Input("vae"), io.Int.Input("width", default=1280, min=32, max=nodes.MAX_RESOLUTION, step=32), @@ -1302,7 +1303,7 @@ class WanInfiniteTalkToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanInfiniteTalkToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/infinite talk", inputs=[ io.DynamicCombo.Input("mode", options=[ io.DynamicCombo.Option("single_speaker", []), @@ -1456,63 +1457,6 @@ class WanInfiniteTalkToVideo(io.ComfyNode): return io.NodeOutput(model_patched, positive, negative, out_latent, trim_image) -class WanSCAILToVideo(io.ComfyNode): - @classmethod - def define_schema(cls): - return io.Schema( - node_id="WanSCAILToVideo", - category="model/conditioning/video_models", - inputs=[ - io.Conditioning.Input("positive"), - io.Conditioning.Input("negative"), - io.Vae.Input("vae"), - io.Int.Input("width", default=512, min=32, max=nodes.MAX_RESOLUTION, step=32), - io.Int.Input("height", default=896, min=32, max=nodes.MAX_RESOLUTION, step=32), - io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4), - io.Int.Input("batch_size", default=1, min=1, max=4096), - io.ClipVisionOutput.Input("clip_vision_output", optional=True), - io.Image.Input("reference_image", optional=True), - io.Image.Input("pose_video", optional=True, tooltip="Video used for pose conditioning. Will be downscaled to half the resolution of the main video."), - io.Float.Input("pose_strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Strength of the pose latent."), - io.Float.Input("pose_start", default=0.0, min=0.0, max=1.0, step=0.01, tooltip="Start step to use pose conditioning."), - io.Float.Input("pose_end", default=1.0, min=0.0, max=1.0, step=0.01, tooltip="End step to use pose conditioning."), - ], - outputs=[ - io.Conditioning.Output(display_name="positive"), - io.Conditioning.Output(display_name="negative"), - io.Latent.Output(display_name="latent", tooltip="Empty latent of the generation size."), - ], - is_experimental=True, - ) - - @classmethod - def execute(cls, positive, negative, vae, width, height, length, batch_size, pose_strength, pose_start, pose_end, reference_image=None, clip_vision_output=None, pose_video=None) -> io.NodeOutput: - latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) - - ref_latent = None - if reference_image is not None: - reference_image = comfy.utils.common_upscale(reference_image[:1].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) - ref_latent = vae.encode(reference_image[:, :, :, :3]) - - if ref_latent is not None: - positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True) - negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [torch.zeros_like(ref_latent)]}, append=True) - - if clip_vision_output is not None: - positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) - negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) - - if pose_video is not None: - pose_video = comfy.utils.common_upscale(pose_video[:length].movedim(-1, 1), width // 2, height // 2, "area", "center").movedim(1, -1) - pose_video_latent = vae.encode(pose_video[:, :, :, :3]) * pose_strength - positive = node_helpers.conditioning_set_values_with_timestep_range(positive, {"pose_video_latent": pose_video_latent}, pose_start, pose_end) - negative = node_helpers.conditioning_set_values_with_timestep_range(negative, {"pose_video_latent": pose_video_latent}, pose_start, pose_end) - - out_latent = {} - out_latent["samples"] = latent - return io.NodeOutput(positive, negative, out_latent) - - class WanExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: @@ -1533,7 +1477,6 @@ class WanExtension(ComfyExtension): WanAnimateToVideo, Wan22ImageToVideoLatent, WanInfiniteTalkToVideo, - WanSCAILToVideo, ] async def comfy_entrypoint() -> WanExtension: diff --git a/comfy_extras/nodes_wandancer.py b/comfy_extras/nodes_wandancer.py index a96885745..fdb2b5e57 100644 --- a/comfy_extras/nodes_wandancer.py +++ b/comfy_extras/nodes_wandancer.py @@ -713,7 +713,7 @@ class WanDancerEncodeAudio(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanDancerEncodeAudio", - category="model/conditioning/video_models", + category="model/conditioning/wan/dancer", inputs=[ io.Audio.Input("audio"), io.Int.Input("video_frames", default=149, min=1, max=nodes.MAX_RESOLUTION, step=4), @@ -787,7 +787,7 @@ class WanDancerVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanDancerVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/dancer", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), diff --git a/comfy_extras/nodes_wanmove.py b/comfy_extras/nodes_wanmove.py index 2db064922..d1f924a40 100644 --- a/comfy_extras/nodes_wanmove.py +++ b/comfy_extras/nodes_wanmove.py @@ -247,7 +247,7 @@ class WanMoveVisualizeTracks(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveVisualizeTracks", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Image.Input("images"), io.Tracks.Input("tracks", optional=True), @@ -283,7 +283,7 @@ class WanMoveTracksFromCoords(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveTracksFromCoords", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.String.Input("track_coords", force_input=True, default="[]", optional=True), io.Mask.Input("track_mask", optional=True), @@ -325,7 +325,8 @@ class GenerateTracks(io.ComfyNode): return io.Schema( node_id="GenerateTracks", search_aliases=["motion paths", "camera movement", "trajectory"], - category="model/conditioning/video_models", + display_name="Generate Video Tracks", + category="model/conditioning/wan/move", inputs=[ io.Int.Input("width", default=832, min=16, max=4096, step=16), io.Int.Input("height", default=480, min=16, max=4096, step=16), @@ -434,7 +435,7 @@ class WanMoveConcatTrack(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveConcatTrack", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Tracks.Input("tracks_1"), io.Tracks.Input("tracks_2", optional=True), @@ -463,7 +464,7 @@ class WanMoveTrackToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveTrackToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), diff --git a/comfy_extras/nodes_zimage.py b/comfy_extras/nodes_zimage.py index 70ddc4afa..ce946b377 100644 --- a/comfy_extras/nodes_zimage.py +++ b/comfy_extras/nodes_zimage.py @@ -10,7 +10,7 @@ class TextEncodeZImageOmni(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeZImageOmni", - category="advanced/conditioning", + category="model/conditioning/z-image", is_experimental=True, inputs=[ io.Clip.Input("clip"), diff --git a/comfyui_version.py b/comfyui_version.py index 19e8f8cfc..dcc0fee96 100644 --- a/comfyui_version.py +++ b/comfyui_version.py @@ -1,3 +1,3 @@ # This file is automatically generated by the build process when version is # updated in pyproject.toml. -__version__ = "0.23.0" +__version__ = "0.28.0" diff --git a/cuda_malloc.py b/cuda_malloc.py index f7651981c..8c4422db8 100644 --- a/cuda_malloc.py +++ b/cuda_malloc.py @@ -2,6 +2,7 @@ import os import importlib.util from comfy.cli_args import args, PerformanceFeature import subprocess +import re #Can't use pytorch to get the GPU names because the cuda malloc has to be set before the first import. def get_gpu_names(): @@ -77,11 +78,24 @@ try: except: pass +def get_raw_cuda_version(version_str): + match = re.search(r'\+cu(\d+)', version_str) + if match: + try: + return int(match.group(1)) + except: + pass + return None + if not args.cuda_malloc: try: if int(version[0]) >= 2 and "+cu" in version: # enable by default for torch version 2.0 and up only on cuda torch if PerformanceFeature.AutoTune not in args.fast: # Autotune has issues with cuda malloc - args.cuda_malloc = cuda_malloc_supported() + cuda_version = get_raw_cuda_version(version) + if cuda_version is not None and cuda_version >= 130: + args.cuda_malloc = True + else: + args.cuda_malloc = cuda_malloc_supported() except: pass diff --git a/execution.py b/execution.py index 5246d651c..387772629 100644 --- a/execution.py +++ b/execution.py @@ -29,6 +29,7 @@ from comfy_execution.caching import ( HierarchicalCache, LRUCache, RAMPressureCache, + RAM_CACHE_LARGE_INTERMEDIATE, ) from comfy_execution.graph import ( DynamicPrompt, @@ -40,6 +41,7 @@ from comfy_execution.graph_utils import GraphBuilder, is_link from comfy_execution.validation import validate_node_input from comfy_execution.progress import get_progress_state, reset_progress_state, add_progress_handler, WebUIProgressHandler from comfy_execution.utils import CurrentNodeContext +from comfy_execution.asset_enrichment import enrich_output_with_assets from comfy_api.internal import _ComfyNodeInternal, _NodeOutputInternal, first_real_override, is_class, make_locked_method_func from comfy_api.latest import io, _io from comfy_execution.cache_provider import _has_cache_providers, _get_cache_providers, _logger as _cache_logger @@ -199,6 +201,8 @@ def get_input_data(inputs, class_def, unique_id, execution_list=None, dynprompt= hidden_inputs_v3[io.Hidden.auth_token_comfy_org] = extra_data.get("auth_token_comfy_org", None) if io.Hidden.api_key_comfy_org.name in hidden: hidden_inputs_v3[io.Hidden.api_key_comfy_org] = extra_data.get("api_key_comfy_org", None) + if io.Hidden.comfy_usage_source.name in hidden: + hidden_inputs_v3[io.Hidden.comfy_usage_source] = extra_data.get("comfy_usage_source", None) else: if "hidden" in valid_inputs: h = valid_inputs["hidden"] @@ -215,6 +219,8 @@ def get_input_data(inputs, class_def, unique_id, execution_list=None, dynprompt= input_data_all[x] = [extra_data.get("auth_token_comfy_org", None)] if h[x] == "API_KEY_COMFY_ORG": input_data_all[x] = [extra_data.get("api_key_comfy_org", None)] + if h[x] == "COMFY_USAGE_SOURCE": + input_data_all[x] = [extra_data.get("comfy_usage_source", None)] v3_data["hidden_inputs"] = hidden_inputs_v3 return input_data_all, missing_keys, v3_data @@ -418,13 +424,14 @@ def _is_intermediate_output(dynprompt, node_id): class_def = nodes.NODE_CLASS_MAPPINGS[class_type] return getattr(class_def, 'HAS_INTERMEDIATE_OUTPUT', False) + def _send_cached_ui(server, node_id, display_node_id, cached, prompt_id, ui_outputs): + if cached.ui is not None: + ui_outputs[node_id] = cached.ui if server.client_id is None: return cached_ui = cached.ui or {} server.send_sync("executed", { "node": node_id, "display_node": display_node_id, "output": cached_ui.get("output", None), "prompt_id": prompt_id }, server.client_id) - if cached.ui is not None: - ui_outputs[node_id] = cached.ui async def execute(server, dynprompt, caches, current_item, extra_data, executed, prompt_id, execution_list, pending_subgraph_results, pending_async_nodes, ui_outputs): unique_id = current_item @@ -552,6 +559,10 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed, asyncio.create_task(await_completion()) return (ExecutionResult.PENDING, None, None) if len(output_ui) > 0: + # Enrich at output-processing time (not in the send path) so assets + # are registered even when no client is connected, and the asset id + # flows into ui_outputs and the cache alongside the raw entries. + output_ui = enrich_output_with_assets(output_ui) ui_outputs[unique_id] = { "meta": { "node_id": unique_id, @@ -784,12 +795,16 @@ class PromptExecutor: if self.cache_type == CacheType.RAM_PRESSURE: ram_release_callback(ram_inactive_headroom) ram_shortfall = ram_headroom - psutil.virtual_memory().available - freed = comfy.model_management.free_pins(ram_shortfall + 512 * (1024 ** 2)) - if freed < ram_shortfall: - if freed > 64 * (1024 ** 2): - # AIMDO MEM_DECOMMIT can outrun psutil.available catching up. - time.sleep(0.05) - ram_release_callback(ram_headroom, free_active=True) + if ram_shortfall > 0: + freed = ram_release_callback(ram_headroom, free_active=True, min_entry_size=RAM_CACHE_LARGE_INTERMEDIATE) + ram_shortfall -= freed + if comfy.model_management.should_free_pins_for_ram_pressure(ram_shortfall): + freed = comfy.model_management.free_pins(ram_shortfall + 512 * (1024 ** 2)) + if freed < ram_shortfall: + if freed > 64 * (1024 ** 2): + # AIMDO MEM_DECOMMIT can outrun psutil.available catching up. + time.sleep(0.05) + ram_release_callback(ram_headroom, free_active=True) else: # Only execute when the while-loop ends without break # Send cached UI for intermediate output nodes that weren't executed @@ -1298,6 +1313,25 @@ class PromptQueue: queued = copy.copy(self.queue) return (running, queued) + def interrupt_if_running(self, prompt_id): + """Interrupt the running prompt with this id, atomically. + + Checks the live running set and signals the interrupt under the queue + mutex, so the worker cannot move the job to done (and start the next + prompt) in between. Returns True if a matching job was running and an + interrupt was signalled, False otherwise. The atomicity is what keeps a + cancel from landing on an unrelated prompt that started after a separate + is-running check: the global interrupt flag is reset at the start of + every prompt (execute_async), so a job that finishes before consuming + the flag cannot leak the interrupt onto its successor. + """ + with self.mutex: + for item in self.currently_running.values(): + if item[1] == prompt_id: + nodes.interrupt_processing() + return True + return False + def get_tasks_remaining(self): with self.mutex: return len(self.queue) + len(self.currently_running) diff --git a/extra_model_paths.yaml.example b/extra_model_paths.yaml.example index 9c395c0b2..6a31d8a63 100644 --- a/extra_model_paths.yaml.example +++ b/extra_model_paths.yaml.example @@ -8,21 +8,37 @@ # # You can use is_default to mark that these folders should be listed first, and used as the default dirs for eg downloads # #is_default: true # checkpoints: models/checkpoints/ +# configs: models/configs/ +# loras: models/loras/ +# vae: models/vae/ # text_encoders: | # models/text_encoders/ -# models/clip/ # legacy location still supported -# clip_vision: models/clip_vision/ -# configs: models/configs/ -# controlnet: models/controlnet/ +# models/clip/ # diffusion_models: | -# models/diffusion_models -# models/unet +# models/unet/ +# models/diffusion_models/ +# clip_vision: models/clip_vision/ +# style_models: models/style_models/ # embeddings: models/embeddings/ -# loras: models/loras/ +# diffusers: models/diffusers/ +# vae_approx: models/vae_approx/ +# controlnet: | +# models/controlnet/ +# models/t2i_adapter/ +# gligen: models/gligen/ # upscale_models: models/upscale_models/ -# vae: models/vae/ -# audio_encoders: models/audio_encoders/ +# latent_upscale_models: models/latent_upscale_models/ +# custom_nodes: custom_nodes/ +# hypernetworks: models/hypernetworks/ +# photomaker: models/photomaker/ +# classifiers: models/classifiers/ # model_patches: models/model_patches/ +# audio_encoders: models/audio_encoders/ +# background_removal: models/background_removal/ +# frame_interpolation: models/frame_interpolation/ +# geometry_estimation: models/geometry_estimation/ +# optical_flow: models/optical_flow/ +# detection: models/detection/ #config for a1111 ui @@ -45,8 +61,7 @@ # controlnet: models/ControlNet -# For a full list of supported keys (style_models, vae_approx, hypernetworks, photomaker, -# model_patches, audio_encoders, classifiers, etc.) see folder_paths.py. +# For the canonical list of supported keys and extensions, see folder_paths.py. #other_ui: # base_path: path/to/ui diff --git a/folder_paths.py b/folder_paths.py index 7304e1b73..937428c18 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -17,7 +17,11 @@ if args.base_directory: else: base_path = os.path.dirname(os.path.realpath(__file__)) -models_dir = os.path.join(base_path, "models") +if args.models_directory: + models_dir = os.path.abspath(args.models_directory) +else: + models_dir = os.path.join(base_path, "models") + folder_names_and_paths["checkpoints"] = ([os.path.join(models_dir, "checkpoints")], supported_pt_extensions) folder_names_and_paths["configs"] = ([os.path.join(models_dir, "configs")], [".yaml"]) @@ -264,6 +268,59 @@ def annotated_filepath(name: str) -> tuple[str, str | None]: return name, base_dir +# Content types a browser may execute or render inline. File endpoints that +# serve user-controlled content must force these to download (and ideally set +# Content-Disposition: attachment) to avoid stored XSS. Centralised here so the +# /view and /userdata handlers can't drift apart. mimetypes.guess_type may +# return either the text/* or application/* spelling depending on platform, so +# both are listed. +DANGEROUS_CONTENT_TYPES = { + 'text/html', 'text/html-sandboxed', 'application/xhtml+xml', + 'text/javascript', 'application/javascript', 'application/x-javascript', + 'application/ecmascript', 'text/css', + 'image/svg+xml', 'application/xml', 'text/xml', + # message/rfc822 (.mht/.mhtml) can carry script in some browsers. + 'message/rfc822', +} + + +def is_dangerous_content_type(content_type: str | None) -> bool: + """Return True if a browser may execute or render `content_type` inline. + + Normalises before matching so the check can't be slipped past with a + charset/boundary parameter (``text/html; charset=utf-8``) or casing + (``TEXT/HTML``). Any XML dialect (``*+xml`` or ``*/xml``) is treated as + dangerous because XML can carry inline script via stylesheet/entity tricks, + which also covers the ``application/{xslt,rss,atom,rdf}+xml`` family without + enumerating each one. Endpoints serving user-controlled content should route + a dangerous type to ``application/octet-stream`` + ``Content-Disposition: + attachment`` + ``X-Content-Type-Options: nosniff``. + """ + if not content_type: + return False + normalized = content_type.split(';', 1)[0].strip().lower() + if normalized in DANGEROUS_CONTENT_TYPES: + return True + return normalized.endswith('+xml') or normalized.endswith('/xml') + + +def is_within_directory(directory: str, target: str) -> bool: + """Return True if `target` resolves to a path inside `directory`. + + Uses realpath on both operands so that a symlink placed inside `directory` + that points elsewhere cannot escape the containment check at open time. + """ + try: + directory = os.path.realpath(directory) + target = os.path.realpath(target) + return os.path.commonpath((directory, target)) == directory + except ValueError: + # ValueError is raised by realpath() on a path with an embedded null + # byte, and by commonpath() on Windows when the paths are on different + # drives. In either case the target is not safely within the directory. + return False + + def get_annotated_filepath(name: str, default_dir: str | None=None) -> str: name, base_dir = annotated_filepath(name) @@ -273,7 +330,12 @@ def get_annotated_filepath(name: str, default_dir: str | None=None) -> str: else: base_dir = get_input_directory() # fallback path - return os.path.join(base_dir, name) + filepath = os.path.abspath(os.path.join(base_dir, name)) + # Prevent path traversal: the resolved path must stay within base_dir. + # repr() the name in the message so a crafted value can't inject log lines. + if not is_within_directory(base_dir, filepath): + raise ValueError("Invalid file path: {!r}".format(name)) + return filepath def exists_annotated_filepath(name) -> bool: @@ -282,7 +344,10 @@ def exists_annotated_filepath(name) -> bool: if base_dir is None: base_dir = get_input_directory() # fallback path - filepath = os.path.join(base_dir, name) + filepath = os.path.abspath(os.path.join(base_dir, name)) + # Treat traversal attempts as non-existent rather than probing the filesystem. + if not is_within_directory(base_dir, filepath): + return False return os.path.exists(filepath) diff --git a/main.py b/main.py index 239a52013..580074b19 100644 --- a/main.py +++ b/main.py @@ -26,6 +26,7 @@ import utils.extra_config from utils.mime_types import init_mime_types import faulthandler import logging +import signal import sys from comfy_execution.progress import get_progress_state from comfy_execution.utils import get_executing_context @@ -37,12 +38,28 @@ if __name__ == "__main__": os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1' os.environ['DO_NOT_TRACK'] = '1' -faulthandler.enable(file=sys.stderr, all_threads=False) +faulthandler.enable(file=sys.stderr, all_threads=args.debug_hang) +if __name__ == "__main__" and args.debug_hang: + dumping_traceback = False + + def dump_traceback_on_sigint(signum, frame): + global dumping_traceback + if dumping_traceback: + raise KeyboardInterrupt + dumping_traceback = True + faulthandler.dump_traceback(file=sys.stderr, all_threads=True) + raise KeyboardInterrupt + + signal.signal(signal.SIGINT, dump_traceback_on_sigint) import comfy_aimdo.control if enables_dynamic_vram(): - comfy_aimdo.control.init() + try: + comfy_aimdo.control.init(simple_vram_headroom=None if args.reserve_vram is None else int(args.reserve_vram * 1024 ** 3)) + except TypeError: + # comfy-aimdo 0.4.9 protocol. + comfy_aimdo.control.init() if os.name == "nt": os.environ['MIMALLOC_PURGE_DELAY'] = '0' @@ -110,6 +127,14 @@ def apply_custom_paths(): for config_path in itertools.chain(*args.extra_model_paths_config): utils.extra_config.load_extra_path_config(config_path) + # --base-directory + if args.base_directory: + logging.info(f"Setting base directory to: {folder_paths.base_path}") + + # --models-directory + if args.models_directory: + logging.info(f"Setting models directory to: {folder_paths.models_dir}") + # --output-directory, --input-directory, --user-directory if args.output_directory: output_dir = os.path.abspath(args.output_directory) @@ -218,23 +243,30 @@ import comfy.model_patcher if args.enable_dynamic_vram or (enables_dynamic_vram() and comfy.model_management.is_nvidia() and not comfy.model_management.is_wsl()): if (not args.enable_dynamic_vram) and (comfy.model_management.torch_version_numeric < (2, 8)): logging.warning("Unsupported Pytorch detected. DynamicVRAM support requires Pytorch version 2.8 or later. Falling back to legacy ModelPatcher. VRAM estimates may be unreliable especially on Windows") - elif comfy_aimdo.control.init_devices(d.index for d in comfy.model_management.get_all_torch_devices()): - if args.verbose == 'DEBUG': - comfy_aimdo.control.set_log_debug() - elif args.verbose == 'CRITICAL': - comfy_aimdo.control.set_log_critical() - elif args.verbose == 'ERROR': - comfy_aimdo.control.set_log_error() - elif args.verbose == 'WARNING': - comfy_aimdo.control.set_log_warning() - else: #INFO - comfy_aimdo.control.set_log_info() - - comfy.model_patcher.CoreModelPatcher = comfy.model_patcher.ModelPatcherDynamic - comfy.memory_management.aimdo_enabled = True - logging.info("DynamicVRAM support detected and enabled") else: - logging.warning("No working comfy-aimdo install detected. DynamicVRAM support disabled. Falling back to legacy ModelPatcher. VRAM estimates may be unreliable especially on Windows") + try: + aimdo_initialized = comfy_aimdo.control.init_devices((d.index, int(args.vram_headroom * 1024 ** 3)) for d in comfy.model_management.get_all_torch_devices()) + except TypeError: + # comfy-aimdo 0.4.9 protocol. + aimdo_initialized = comfy_aimdo.control.init_devices(d.index for d in comfy.model_management.get_all_torch_devices()) + + if aimdo_initialized: + if args.verbose == 'DEBUG': + comfy_aimdo.control.set_log_debug() + elif args.verbose == 'CRITICAL': + comfy_aimdo.control.set_log_critical() + elif args.verbose == 'ERROR': + comfy_aimdo.control.set_log_error() + elif args.verbose == 'WARNING': + comfy_aimdo.control.set_log_warning() + else: #INFO + comfy_aimdo.control.set_log_info() + + comfy.model_patcher.CoreModelPatcher = comfy.model_patcher.ModelPatcherDynamic + comfy.memory_management.aimdo_enabled = True + logging.info("DynamicVRAM support detected and enabled") + else: + logging.warning("No working comfy-aimdo install detected. DynamicVRAM support disabled. Falling back to legacy ModelPatcher. VRAM estimates may be unreliable especially on Windows") def cuda_malloc_warning(): @@ -375,7 +407,7 @@ def prompt_worker(q, server_instance): hook_breaker_ac10a0.restore_functions() if not asset_seeder.is_disabled(): - asset_seeder.enqueue_enrich(roots=("output",), compute_hashes=True) + asset_seeder.enqueue_enrich(roots=("output",), compute_hashes=args.enable_asset_hashing) asset_seeder.resume() @@ -430,7 +462,7 @@ def setup_database(): if dependencies_available(): init_db() if args.enable_assets: - if asset_seeder.start(roots=("models", "input", "output"), prune_first=True, compute_hashes=True): + if asset_seeder.start(roots=("models", "input", "output"), prune_first=True, compute_hashes=args.enable_asset_hashing): logging.info("Background asset scan initiated for models, input, output") except Exception as e: if "database is locked" in str(e): @@ -477,6 +509,11 @@ def start_comfyui(asyncio_loop=None): init_custom_nodes=(not args.disable_all_custom_nodes) or len(args.whitelist_custom_nodes) > 0, init_api_nodes=not args.disable_api_nodes )) + + # Re-apply Comfy's cuDNN benchmark policy after custom-node imports. Benchmark + # mode can request near-card-sized autotune workspaces, and some custom nodes set it at import time. + comfy.model_management.set_cudnn_benchmark() + hook_breaker_ac10a0.restore_functions() cuda_malloc_warning() @@ -524,8 +561,13 @@ if __name__ == "__main__": logging.warning("WARNING: You are using a python version older than 3.10, please upgrade to a newer one. 3.12 and above is recommended.") if args.disable_dynamic_vram: - logging.warning("Dynamic vram disabled with argument. If you have any issues with dynamic vram enabled please give us a detailed reports as this argument will be removed soon.") - + logging.warning( + "Dynamic vram disabled with argument. If you have any issues with " + "dynamic vram enabled please give us a detailed reports as this " + "argument will be removed soon. If you use gguf we recommend keeping " + "dynamic vram enabled and using native ComfyUI model formats instead. " + "ComfyUI native formats like fp8 will be faster even if they are larger than your memory." + ) event_loop, _, start_all_func = start_comfyui() try: x = start_all_func() diff --git a/manager_requirements.txt b/manager_requirements.txt index a079d3492..13786bb35 100644 --- a/manager_requirements.txt +++ b/manager_requirements.txt @@ -1 +1 @@ -comfyui_manager==4.2.1 +comfyui_manager==4.2.2 diff --git a/nodes.py b/nodes.py index 331425b87..883258bd1 100644 --- a/nodes.py +++ b/nodes.py @@ -20,8 +20,6 @@ from PIL.PngImagePlugin import PngInfo import numpy as np import safetensors.torch -sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy")) - import comfy.diffusers_load import comfy.samplers import comfy.sample @@ -87,7 +85,7 @@ class ConditioningCombine: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "combine" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" SEARCH_ALIASES = ["combine", "merge conditioning", "combine prompts", "merge prompts", "mix prompts", "add prompt"] def combine(self, conditioning_1, conditioning_2): @@ -104,7 +102,7 @@ class ConditioningAverage : RETURN_TYPES = ("CONDITIONING",) FUNCTION = "addWeighted" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def addWeighted(self, conditioning_to, conditioning_from, conditioning_to_strength): out = [] @@ -143,7 +141,7 @@ class ConditioningConcat: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "concat" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def concat(self, conditioning_to, conditioning_from): out = [] @@ -161,6 +159,29 @@ class ConditioningConcat: return (out, ) +class ConditioningMultiply: + SEARCH_ALIASES = ["scale conditioning", "scale prompt", "multiply conditioning", "multiply prompt"] + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"conditioning": ("CONDITIONING", ), + "multiplier": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}) + }} + RETURN_TYPES = ("CONDITIONING",) + FUNCTION = "multiply" + CATEGORY = "model/conditioning/transform" + + def multiply(self, conditioning, multiplier): + c = [] + for t in conditioning: + values = {} + pooled_output = t[1].get("pooled_output", None) + if pooled_output is not None: + values["pooled_output"] = pooled_output * multiplier + scaled = node_helpers.conditioning_set_values([[t[0] * multiplier, t[1]]], values)[0] + c.append(scaled) + return (c,) + class ConditioningSetArea: SEARCH_ALIASES = ["regional prompt", "area prompt", "spatial conditioning", "localized prompt"] @@ -176,7 +197,7 @@ class ConditioningSetArea: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, width, height, x, y, strength): c = node_helpers.conditioning_set_values(conditioning, {"area": (height // 8, width // 8, y // 8, x // 8), @@ -197,7 +218,7 @@ class ConditioningSetAreaPercentage: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, width, height, x, y, strength): c = node_helpers.conditioning_set_values(conditioning, {"area": ("percentage", height, width, y, x), @@ -214,7 +235,7 @@ class ConditioningSetAreaStrength: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, strength): c = node_helpers.conditioning_set_values(conditioning, {"strength": strength}) @@ -234,7 +255,7 @@ class ConditioningSetMask: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, mask, set_cond_area, strength): set_area_to_bounds = False @@ -257,7 +278,7 @@ class ConditioningZeroOut: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "zero_out" - CATEGORY = "advanced/conditioning" + CATEGORY = "model/conditioning/transform" def zero_out(self, conditioning): c = [] @@ -283,11 +304,10 @@ class ConditioningSetTimestepRange: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "set_range" - CATEGORY = "advanced/conditioning" + CATEGORY = "model/conditioning/transform" def set_range(self, conditioning, start, end): - c = node_helpers.conditioning_set_values(conditioning, {"start_percent": start, - "end_percent": end}) + c = node_helpers.conditioning_set_values(conditioning, {"start_percent": start, "end_percent": end}) return (c, ) class VAEDecode: @@ -329,7 +349,7 @@ class VAEDecodeTiled: RETURN_TYPES = ("IMAGE",) FUNCTION = "decode" - CATEGORY = "experimental" + CATEGORY = "model/latent" def decode(self, vae, samples, tile_size, overlap=64, temporal_size=64, temporal_overlap=8): if tile_size < overlap * 4: @@ -376,7 +396,7 @@ class VAEEncodeTiled: RETURN_TYPES = ("LATENT",) FUNCTION = "encode" - CATEGORY = "experimental" + CATEGORY = "model/latent" def encode(self, vae, pixels, tile_size, overlap, temporal_size=64, temporal_overlap=8): t = vae.encode_tiled(pixels, tile_x=tile_size, tile_y=tile_size, overlap=overlap, tile_t=temporal_size, overlap_t=temporal_overlap) @@ -389,7 +409,7 @@ class VAEEncodeForInpaint: RETURN_TYPES = ("LATENT",) FUNCTION = "encode" - CATEGORY = "model/latent/inpaint" + CATEGORY = "model/latent" def encode(self, vae, pixels, mask, grow_mask_by=6): downscale_ratio = vae.spacial_compression_encode() @@ -438,7 +458,7 @@ class InpaintModelConditioning: RETURN_NAMES = ("positive", "negative", "latent") FUNCTION = "encode" - CATEGORY = "model/conditioning/inpaint" + CATEGORY = "model/conditioning" def encode(self, positive, negative, pixels, vae, mask, noise_mask=True): x = (pixels.shape[1] // 8) * 8 @@ -483,16 +503,18 @@ class SaveLatent: @classmethod def INPUT_TYPES(s): - return {"required": { "samples": ("LATENT", ), - "filename_prefix": ("STRING", {"default": "latents/ComfyUI"})}, - "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, - } - RETURN_TYPES = () + return { "required": { + "samples": ("LATENT",), + "filename_prefix": ("STRING", {"default": "latents/ComfyUI"})}, + "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, + } + RETURN_TYPES = ("LATENT",) + RETURN_NAMES = ("samples",) FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "experimental" + CATEGORY = "model/latent" def save(self, samples, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None): full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) @@ -525,7 +547,7 @@ class SaveLatent: output["latent_format_version_0"] = torch.tensor([]) comfy.utils.save_torch_file(output, file, metadata=metadata) - return { "ui": { "latents": results } } + return { "ui": { "latents": results }, "result": (samples,) } class LoadLatent: @@ -537,7 +559,7 @@ class LoadLatent: files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f)) and f.endswith(".latent")] return {"required": {"latent": [sorted(files), ]}, } - CATEGORY = "experimental" + CATEGORY = "model/latent" RETURN_TYPES = ("LATENT", ) FUNCTION = "load" @@ -576,7 +598,7 @@ class CheckpointLoader: RETURN_TYPES = ("MODEL", "CLIP", "VAE") FUNCTION = "load_checkpoint" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" DEPRECATED = True def load_checkpoint(self, config_name, ckpt_name): @@ -622,8 +644,9 @@ class DiffusersLoader: return {"required": {"model_path": (paths,), }} RETURN_TYPES = ("MODEL", "CLIP", "VAE") FUNCTION = "load_checkpoint" + DEPRECATED = True - CATEGORY = "advanced/loaders/deprecated" + CATEGORY = "model/loaders" def load_checkpoint(self, model_path, output_vae=True, output_clip=True): for search_path in folder_paths.get_folder_paths("diffusers"): @@ -949,7 +972,7 @@ class UNETLoader: RETURN_TYPES = ("MODEL",) FUNCTION = "load_unet" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" def load_unet(self, unet_name, weight_dtype): model_options = {} @@ -969,7 +992,7 @@ class CLIPLoader: @classmethod def INPUT_TYPES(s): return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ), - "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit"], ), + "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2"], ), }, "optional": { "device": (["default", "cpu"], {"advanced": True}), @@ -977,9 +1000,9 @@ class CLIPLoader: RETURN_TYPES = ("CLIP",) FUNCTION = "load_clip" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" - DESCRIPTION = "[Recipes]\n\nstable_diffusion: clip-l\nstable_cascade: clip-g\nsd3: t5 xxl/ clip-g / clip-l\nstable_audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\n hidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\nlens: gpt-oss-20b\n pixeldit: gemma 2 2B elm" + DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm" def load_clip(self, clip_name, type="stable_diffusion", device="default"): clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) @@ -1005,9 +1028,9 @@ class DualCLIPLoader: RETURN_TYPES = ("CLIP",) FUNCTION = "load_clip" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" - DESCRIPTION = "[Recipes]\n\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5\nhidream: at least one of t5 or llama, recommended t5 and llama\nhunyuan_image: qwen2.5vl 7b and byt5 small\nnewbie: gemma-3-4b-it, jina clip v2" + DESCRIPTION = "Recipes:\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5\nhidream: at least one of t5 or llama, recommended t5 and llama\nhunyuan_image: qwen2.5vl 7b and byt5 small\nnewbie: gemma-3-4b-it, jina clip v2" def load_clip(self, clip_name1, clip_name2, type, device="default"): clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) @@ -1088,7 +1111,7 @@ class StyleModelApply: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "apply_stylemodel" - CATEGORY = "model/conditioning/style_model" + CATEGORY = "model/conditioning" def apply_stylemodel(self, conditioning, style_model, clip_vision_output, strength, strength_type): cond = style_model.get_cond(clip_vision_output).flatten(start_dim=0, end_dim=1).unsqueeze(dim=0) @@ -1518,13 +1541,11 @@ class LatentCrop: class SetLatentNoiseMask: @classmethod def INPUT_TYPES(s): - return {"required": { "samples": ("LATENT",), - "mask": ("MASK",), - }} + return {"required": { "samples": ("LATENT",), "mask": ("MASK",), }} RETURN_TYPES = ("LATENT",) FUNCTION = "set_mask" - CATEGORY = "model/latent/inpaint" + CATEGORY = "model/latent" def set_mask(self, samples, mask): s = samples.copy() @@ -1631,14 +1652,18 @@ class SaveImage: return { "required": { "images": ("IMAGE", {"tooltip": "The images to save."}), - "filename_prefix": ("STRING", {"default": "ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."}) + "filename_prefix": ("STRING", { + "default": "ComfyUI", + "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes." + }) }, "hidden": { "prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO" }, } - RETURN_TYPES = () + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("images",) FUNCTION = "save_images" OUTPUT_NODE = True @@ -1674,7 +1699,7 @@ class SaveImage: }) counter += 1 - return { "ui": { "images": results } } + return { "ui": { "images": results }, "result" : (images,) } class PreviewImage(SaveImage): def __init__(self): @@ -1684,6 +1709,7 @@ class PreviewImage(SaveImage): self.compress_level = 1 SEARCH_ALIASES = ["preview", "preview image", "show image", "view image", "display image", "image viewer"] + DESCRIPTION = "Preview the images without saving them to the ComfyUI output directory." @classmethod def INPUT_TYPES(s): @@ -2045,9 +2071,10 @@ NODE_CLASS_MAPPINGS = { "ImageBatch": ImageBatch, "ImagePadForOutpaint": ImagePadForOutpaint, "EmptyImage": EmptyImage, - "ConditioningAverage": ConditioningAverage , + "ConditioningAverage": ConditioningAverage, "ConditioningCombine": ConditioningCombine, "ConditioningConcat": ConditioningConcat, + "ConditioningMultiply": ConditioningMultiply, "ConditioningSetArea": ConditioningSetArea, "ConditioningSetAreaPercentage": ConditioningSetAreaPercentage, "ConditioningSetAreaStrength": ConditioningSetAreaStrength, @@ -2101,6 +2128,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LoraLoader": "Load LoRA (Model and CLIP)", "LoraLoaderModelOnly": "Load LoRA", "CLIPLoader": "Load CLIP", + "DualCLIPLoader": "Load CLIP (Dual)", "ControlNetLoader": "Load ControlNet Model", "DiffControlNetLoader": "Load ControlNet Model (diff)", "StyleModelLoader": "Load Style Model", @@ -2108,6 +2136,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "UNETLoader": "Load Diffusion Model", "unCLIPCheckpointLoader": "Load unCLIP Checkpoint", "GLIGENLoader": "Load GLIGEN Model", + "DiffusersLoader": "Load Diffusers Model (DEPRECATED)", # Conditioning "CLIPVisionEncode": "CLIP Vision Encode", "StyleModelApply": "Apply Style Model", @@ -2115,13 +2144,20 @@ NODE_DISPLAY_NAME_MAPPINGS = { "CLIPSetLastLayer": "CLIP Set Last Layer", "ConditioningCombine": "Conditioning (Combine)", "ConditioningAverage ": "Conditioning (Average)", + "ConditioningAverage": "Conditioning (Average)", "ConditioningConcat": "Conditioning (Concat)", + "ConditioningMultiply": "Conditioning (Multiply)", "ConditioningSetArea": "Conditioning (Set Area)", "ConditioningSetAreaPercentage": "Conditioning (Set Area with Percentage)", + "ConditioningSetAreaStrength": "Conditioning (Set Area Strength)", "ConditioningSetMask": "Conditioning (Set Mask)", "ControlNetApply": "Apply ControlNet (DEPRECATED)", "ControlNetApplyAdvanced": "Apply ControlNet", + "GLIGENTextBoxApply": "Apply GLIGEN Text Box", + "ConditioningZeroOut": "Conditioning Zero Out", # Latent + "LoadLatent": "Load Latent", + "SaveLatent": "Save Latent", "VAEEncodeForInpaint": "VAE Encode (for Inpainting)", "SetLatentNoiseMask": "Set Latent Noise Mask", "VAEDecode": "VAE Decode", @@ -2134,7 +2170,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LatentUpscaleBy": "Upscale Latent By", "LatentComposite": "Latent Composite", "LatentBlend": "Latent Blend", - "LatentFromBatch" : "Latent From Batch", + "LatentFromBatch" : "Get Latent From Batch", "RepeatLatentBatch": "Repeat Latent Batch", # Image "EmptyImage": "Empty Image", @@ -2156,7 +2192,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ImageSharpen": "Sharpen Image", "ImageScaleToTotalPixels": "Scale Image to Total Pixels", "GetImageSize": "Get Image Size", - # experimental "VAEDecodeTiled": "VAE Decode (Tiled)", "VAEEncodeTiled": "VAE Encode (Tiled)", } @@ -2295,6 +2330,9 @@ async def init_external_custom_nodes(): Returns: None """ + # TODO: remove at some point when custom nodes don't break. + sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy")) + base_node_names = set(NODE_CLASS_MAPPINGS.keys()) node_paths = folder_paths.get_folder_paths("custom_nodes") node_import_times = [] @@ -2362,6 +2400,9 @@ async def init_builtin_extra_nodes(): "nodes_model_downscale.py", "nodes_images.py", "nodes_video_model.py", + "nodes_ideogram4.py", + "nodes_bounding_boxes.py", + "nodes_json_prompt.py", "nodes_train.py", "nodes_dataset.py", "nodes_sag.py", @@ -2403,6 +2444,7 @@ async def init_builtin_extra_nodes(): "nodes_video.py", "nodes_lumina2.py", "nodes_wan.py", + "nodes_bernini.py", "nodes_lotus.py", "nodes_hunyuan3d.py", "nodes_primitive.py", @@ -2417,8 +2459,10 @@ async def init_builtin_extra_nodes(): "nodes_camera_trajectory.py", "nodes_edit_model.py", "nodes_tcfg.py", + "nodes_seedvr.py", "nodes_context_windows.py", "nodes_qwen.py", + "nodes_boogu.py", "nodes_chroma_radiance.py", "nodes_pid.py", "nodes_model_patch.py", @@ -2436,6 +2480,7 @@ async def init_builtin_extra_nodes(): "nodes_glsl.py", "nodes_lora_debug.py", "nodes_textgen.py", + "nodes_text_overlay.py", "nodes_color.py", "nodes_toolkit.py", "nodes_replacements.py", @@ -2449,6 +2494,7 @@ async def init_builtin_extra_nodes(): "nodes_rtdetr.py", "nodes_frame_interpolation.py", "nodes_sam3.py", + "nodes_scail.py", "nodes_void.py", "nodes_wandancer.py", "nodes_hidream_o1.py", @@ -2456,7 +2502,10 @@ async def init_builtin_extra_nodes(): "nodes_moge.py", "nodes_mediapipe.py", "nodes_gaussian_splat.py", - "nodes_triposplat.py" + "nodes_triposplat.py", + "nodes_depth_anything_3.py", + "nodes_seed.py", + "nodes_text.py", ] import_failed = [] diff --git a/openapi.yaml b/openapi.yaml index f801a39d9..e00643bad 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1,11749 +1,5111 @@ -openapi: 3.1.0 -info: - title: ComfyUI API - description: | - API for ComfyUI - A powerful and modular stable diffusion GUI and backend. - - This API allows you to interact with ComfyUI programmatically, including: - - Submitting and managing workflow executions - - Querying node/object information - - Uploading and viewing files - - Managing user settings and data - - Asset management (feature-gated) - - ## Dual-path routing - Every route registered via `self.routes` in the ComfyUI server is available at - both its bare path (e.g. `/prompt`) and an `/api`-prefixed path (e.g. `/api/prompt`). - This spec uses the `/api`-prefixed versions as canonical. - - ## Multi-user mode - When ComfyUI is started with `--multi-user`, the `Comfy-User` header identifies - the active user for settings, userdata, and history isolation. This is **not** a - security mechanism — it is an organisational convenience with no authentication - or authorisation behind it. - version: 1.0.0 - license: - name: GNU General Public License v3.0 - url: https://github.com/comfyanonymous/ComfyUI/blob/master/LICENSE - -servers: - - url: / - description: Default ComfyUI server (typically http://127.0.0.1:8188) - -tags: - - name: prompt - description: Workflow submission and prompt info - - name: queue - description: Queue inspection and management - - name: history - description: Execution history - - name: upload - description: File upload endpoints - - name: view - description: File viewing / download - - name: system - description: System stats and feature flags - - name: node - description: Node / object_info definitions - - name: model - description: Model folder and file listing - - name: user - description: User management (multi-user mode) - - name: userdata - description: Per-user file storage - - name: settings - description: Per-user settings - - name: extensions - description: Frontend extension JS files - - name: subgraph - description: Global subgraph blueprints - - name: internal - description: Internal / debug endpoints - - name: assets - description: Asset management (feature-gated behind enable-assets) - - - name: auth - description: Authentication and session management (cloud-only) - - name: billing - description: Billing, subscriptions, and payment management (cloud-only) - - name: workspace - description: Workspace and team management (cloud-only) - - name: hub - description: "ComfyUI Hub: profiles, shared workflows, and labels (cloud-only)" - - name: workflows - description: Cloud workflow management and versioning (cloud-only) - - name: task - description: Background task management (cloud-only) - - name: runtime-only - description: Operations served exclusively by the cloud runtime with no local equivalent - -paths: - # --------------------------------------------------------------------------- - # WebSocket - # --------------------------------------------------------------------------- - /ws: - get: - operationId: connectWebSocket - tags: [system] - summary: WebSocket connection for real-time updates - description: | - Upgrades to a WebSocket connection that streams execution progress, - node status, and output messages. The server sends an initial `status` - message with the session ID (SID) on connect. - - ## Message types (server → client) - The server sends JSON messages with a `type` field. See the - `x-websocket-messages` list below for the schema of each message type. - parameters: - - name: clientId - in: query - required: false - schema: - type: string - description: Client identifier. If omitted the server assigns one. - responses: - "101": - description: WebSocket upgrade successful - '401': - description: Unauthorized - x-websocket-messages: - - type: status - schema: - $ref: "#/components/schemas/StatusWsMessage" - - type: progress - schema: - $ref: "#/components/schemas/ProgressWsMessage" - - type: progress_text - schema: - $ref: "#/components/schemas/ProgressTextWsMessage" - - type: progress_state - schema: - $ref: "#/components/schemas/ProgressStateWsMessage" - - type: executing - schema: - $ref: "#/components/schemas/ExecutingWsMessage" - - type: executed - schema: - $ref: "#/components/schemas/ExecutedWsMessage" - - type: execution_start - schema: - $ref: "#/components/schemas/ExecutionStartWsMessage" - - type: execution_success - schema: - $ref: "#/components/schemas/ExecutionSuccessWsMessage" - - type: execution_cached - schema: - $ref: "#/components/schemas/ExecutionCachedWsMessage" - - type: execution_interrupted - schema: - $ref: "#/components/schemas/ExecutionInterruptedWsMessage" - - type: execution_error - schema: - $ref: "#/components/schemas/ExecutionErrorWsMessage" - - type: logs - schema: - $ref: "#/components/schemas/LogsWsMessage" - - type: notification - schema: - $ref: "#/components/schemas/NotificationWsMessage" - - type: feature_flags - schema: - $ref: "#/components/schemas/FeatureFlagsWsMessage" - - type: asset_download - schema: - $ref: "#/components/schemas/AssetDownloadWsMessage" - - type: asset_export - schema: - $ref: "#/components/schemas/AssetExportWsMessage" - - # --------------------------------------------------------------------------- - # Prompt - # --------------------------------------------------------------------------- - /api/prompt: - get: - operationId: getPromptInfo - tags: [prompt] - summary: Get queue status - description: Returns how many items remain in the execution queue. - responses: - "200": - description: Queue info - content: - application/json: - schema: - $ref: "#/components/schemas/PromptInfo" - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: executePrompt - tags: [prompt] - summary: Submit a workflow for execution - description: Submits a workflow for execution. The server validates the graph, assigns a `prompt_id`, and enqueues it. Clients listen on `/ws` for execution progress and output messages. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/PromptRequest" - responses: - "200": - description: Prompt accepted - content: - application/json: - schema: - $ref: "#/components/schemas/PromptResponse" - "400": - description: Validation or node errors - content: - application/json: - schema: - $ref: "#/components/schemas/PromptErrorResponse" - - '402': - description: Payment required - Insufficient credits - content: - application/json: - schema: - $ref: '#/components/schemas/PromptErrorResponse' - '429': - description: Payment required - User has not paid - content: - application/json: - schema: - $ref: '#/components/schemas/PromptErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/PromptErrorResponse' - '503': - description: Service unavailable - content: - application/json: - schema: - $ref: '#/components/schemas/PromptErrorResponse' - # --------------------------------------------------------------------------- - # Queue - # --------------------------------------------------------------------------- - /api/queue: - get: - operationId: getQueueInfo - tags: [queue] - summary: Get running and pending queue items - description: Returns the server's current execution queue, split into the currently-running prompt and the list of pending prompts. - responses: - "200": - description: Queue contents - content: - application/json: - schema: - $ref: "#/components/schemas/QueueInfo" - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: manageQueue - tags: [queue] - summary: Clear or delete items from the queue - description: Mutates the execution queue. Supports clearing all queued prompts or deleting individual prompts by ID. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/QueueManageRequest" - responses: - "200": - description: Queue updated - content: - application/json: - schema: - $ref: "#/components/schemas/QueueManageResponse" - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/interrupt: - post: - operationId: interruptJob - tags: [queue] - summary: Interrupt current execution - description: Interrupts the prompt that is currently executing. The next queued prompt (if any) will start immediately after. - requestBody: - required: false - content: - application/json: - schema: - type: object - properties: - prompt_id: - type: string - format: uuid - description: "If provided, only interrupts this specific running prompt. Otherwise interrupts all." - responses: - "200": - description: Interrupt signal sent - - '401': - description: Unauthorized - Authentication required - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/free: - post: - operationId: freeMemory - tags: [queue] - summary: Free GPU memory and/or unload models - description: Frees GPU memory by unloading models and/or freeing the resident model cache, controlled by the request flags. - requestBody: - required: false - content: - application/json: - schema: - type: object - properties: - unload_models: - type: boolean - description: Unload all models from VRAM/RAM - free_memory: - type: boolean - description: Run garbage collection and free cached memory - responses: - "200": - description: Memory freed - - # --------------------------------------------------------------------------- - # Jobs - # --------------------------------------------------------------------------- - /api/jobs: - get: - operationId: listJobs - tags: [queue] - summary: List jobs with filtering and pagination - description: Returns a paginated list of completed prompt executions, newest first. - parameters: - - name: status - in: query - schema: - type: string - description: Filter by job status - - name: workflow_id - in: query - schema: - type: string - description: Filter by workflow ID - - name: sort_by - in: query - schema: - type: string - description: Field to sort by - - name: sort_order - in: query - schema: - type: string - enum: [asc, desc] - description: Sort direction - - name: limit - in: query - schema: - type: integer - description: Maximum number of results (default is unlimited/None) - - name: offset - in: query - schema: - type: integer - default: 0 - description: Pagination offset - responses: - "200": - description: Jobs list - content: - application/json: - schema: - type: object - properties: - jobs: - type: array - items: - $ref: "#/components/schemas/JobEntry" - pagination: - $ref: "#/components/schemas/PaginationInfo" - - '401': - description: Unauthorized - Authentication required - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/jobs/{job_id}: - get: - operationId: getJobDetail - tags: [queue] - summary: Get a single job by ID - description: Returns the full record for a single completed prompt execution, including its outputs, status, and metadata. - parameters: - - name: job_id - in: path - description: The job (prompt) ID to fetch. - required: true - schema: - type: string - format: uuid - responses: - "200": - description: Job detail - content: - application/json: - schema: - $ref: "#/components/schemas/JobDetailResponse" - "404": - description: Job not found - - '401': - description: Unauthorized - Authentication required - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '403': - description: Forbidden - Job does not belong to user - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # History - # --------------------------------------------------------------------------- - /api/history: - get: - operationId: getPromptHistory - tags: [history] - summary: Get execution history - deprecated: true - description: | - **Deprecated.** Superseded by `GET /api/jobs`, which returns the same - execution records in a paginated, filterable format. Planned for removal - no earlier than a future major release; sunset timeline TBD. - - Returns a dictionary keyed by prompt_id. Each value is a HistoryEntry - containing prompt metadata, outputs, status, and node meta. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: max_items - in: query - schema: - type: integer - description: Maximum number of history entries to return - - name: offset - in: query - schema: - type: integer - description: Pagination offset (number of entries to skip) - responses: - "200": - description: History dictionary keyed by prompt_id - content: - application/json: - schema: - type: object - additionalProperties: - $ref: "#/components/schemas/HistoryEntry" - '404': - description: "Not Found \u2014 use /api/history_v2 instead" - post: - operationId: manageHistory - tags: [history] - summary: Clear or delete history entries - deprecated: true - description: | - **Deprecated.** Superseded by the forthcoming job-management endpoints - under `/api/jobs`. Planned for removal no earlier than a future major - release; sunset timeline TBD. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/HistoryManageRequest" - responses: - "200": - description: History updated - - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - Authentication required - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/history/{prompt_id}: - get: - operationId: getHistoryByPromptId - tags: [history] - summary: Get history for a specific prompt - deprecated: true - description: | - **Deprecated.** Superseded by `GET /api/jobs/{job_id}`, which returns - the same execution record. Planned for removal no earlier than a future - major release; sunset timeline TBD. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: prompt_id - in: path - description: The prompt ID to fetch history for. - required: true - schema: - type: string - format: uuid - responses: - "200": - description: Single-entry history dictionary. Returns an empty object `{}` if the prompt_id is not found. - content: - application/json: - schema: - type: object - additionalProperties: - $ref: "#/components/schemas/HistoryEntry" - - '404': - description: "Not Found \u2014 use /api/jobs/{prompt_id} instead" - # --------------------------------------------------------------------------- - # Upload - # --------------------------------------------------------------------------- - /api/upload/image: - post: - operationId: uploadImage - tags: [upload] - summary: Upload an image file - description: Uploads an image file into one of the input/output/temp directories so it can be referenced by workflow nodes. - requestBody: - required: true - content: - multipart/form-data: - schema: - type: object - required: - - image - properties: - image: - type: string - format: binary - description: Image file to upload - type: - type: string - enum: [input, temp, output] - default: input - description: Target directory type - overwrite: - type: string - description: 'Set to "true" to overwrite existing files' - subfolder: - type: string - description: Subfolder within the target directory - responses: - "200": - description: Upload result - content: - application/json: - schema: - $ref: "#/components/schemas/UploadResult" - "400": - description: No file provided or invalid request - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/upload/mask: - post: - operationId: uploadMask - tags: [upload] - deprecated: true - summary: Upload a mask image (deprecated) - description: | - Deprecated. Clients should composite the mask onto the source image - client-side and upload the resulting image via POST /api/upload/image - instead. This endpoint will continue to function for older clients, - but will not receive new features. - - Uploads a mask image associated with a previously-uploaded reference image. - requestBody: - required: true - content: - multipart/form-data: - schema: - type: object - required: - - image - - original_ref - properties: - image: - type: string - format: binary - description: Mask image (alpha channel is used) - original_ref: - type: object - description: Reference to the original image file - required: - - filename - properties: - filename: - type: string - description: Filename of the original image - additionalProperties: true - type: - type: string - enum: [input, temp, output] - default: input - description: Target directory type - overwrite: - type: string - description: 'Set to "true" to overwrite existing files' - subfolder: - type: string - description: Subfolder within the target directory - responses: - "200": - description: Upload result - content: - application/json: - schema: - $ref: "#/components/schemas/UploadResult" - "400": - description: No file provided or invalid request - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # View - # --------------------------------------------------------------------------- - /api/view: - get: - operationId: viewFile - tags: [view] - summary: View or download a file - description: Serves a file (image, audio, or video) from the input/output/temp directory identified by the query parameters. - parameters: - - name: filename - in: query - required: true - schema: - type: string - description: Name of the file to view - - name: type - in: query - schema: - type: string - enum: [input, output, temp] - default: output - description: Directory type - - name: subfolder - in: query - schema: - type: string - description: Subfolder within the directory - - name: preview - in: query - schema: - type: string - description: Preview format hint (e.g. "webp;90") - - name: channel - in: query - schema: - type: string - enum: [rgba, rgb, a] - description: Channel extraction mode - responses: - "200": - description: File content - content: - image/*: - schema: - type: string - format: binary - video/*: - schema: - type: string - format: binary - audio/*: - schema: - type: string - format: binary - application/octet-stream: - schema: - type: string - format: binary - "404": - description: File not found - - '302': - description: Redirect to GCS signed URL - headers: - Location: - description: Signed URL to access the file in GCS - schema: - type: string - Cache-Control: - description: Cache directive for the redirect response - schema: - type: string - Vary: - description: Headers that affect response caching - schema: - type: string - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/view_metadata/{folder_name}: - get: - operationId: viewMetadata - tags: [view] - summary: Get metadata for a file (e.g. safetensors header) - description: Returns embedded metadata parsed from a file in the given folder — for example, the header of a safetensors model. - parameters: - - name: folder_name - in: path - required: true - schema: - type: string - description: Folder type (output, input, temp, etc.) - - name: filename - in: query - required: true - schema: - type: string - description: Filename to read metadata from - responses: - "200": - description: File metadata - content: - application/json: - schema: - type: object - additionalProperties: true - "404": - description: File or metadata not found - - # --------------------------------------------------------------------------- - # System - # --------------------------------------------------------------------------- - /api/system_stats: - get: - operationId: getSystemStats - tags: [system] - summary: Get system statistics - description: Returns hardware, Python, VRAM, and runtime statistics for the running ComfyUI process. - responses: - "200": - description: System stats - content: - application/json: - schema: - $ref: "#/components/schemas/SystemStatsResponse" - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/features: - get: - operationId: getFeatures - tags: [system] - summary: Get enabled feature flags - description: Returns a dictionary of feature flag names to their enabled state. Cloud deployments may include additional typed fields alongside the boolean flags. - responses: - "200": - description: Feature flags - content: - application/json: - schema: - type: object - additionalProperties: - type: boolean - properties: - max_upload_size: - type: integer - format: int64 - minimum: 0 - description: "Maximum file upload size in bytes." - free_tier_credits: - type: integer - format: int32 - minimum: 0 - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Credits available to free-tier users. Local ComfyUI returns null." - posthog_api_host: - type: string - format: uri - nullable: true - x-runtime: [cloud] - description: "[cloud-only] PostHog analytics proxy URL for frontend telemetry. Local ComfyUI returns null." - max_concurrent_jobs: - type: integer - format: int32 - minimum: 0 - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Maximum concurrent jobs the authenticated user can run. Local ComfyUI returns null." - workflow_templates_version: - type: string - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Version identifier for the workflow templates bundle. Local ComfyUI returns null." - workflow_templates_source: - type: string - nullable: true - enum: [dynamic_config_override, workflow_templates_version_json] - x-runtime: [cloud] - description: "[cloud-only] How the templates version was resolved. Local ComfyUI returns null." - - # --------------------------------------------------------------------------- - # Node / Object Info - # --------------------------------------------------------------------------- - /api/object_info: - get: - operationId: getNodeInfo - tags: [node] - summary: Get all node definitions - description: | - Returns a dictionary of every registered node class, keyed by class name. - Each value is a NodeInfo object describing inputs, outputs, category, etc. - responses: - "200": - description: All node definitions - content: - application/json: - schema: - type: object - additionalProperties: - $ref: "#/components/schemas/NodeInfo" - - /api/object_info/{node_class}: - get: - operationId: getObjectInfoByClass - tags: [node] - summary: Get a single node definition - description: Returns the `NodeInfo` definition for a single registered node class. - parameters: - - name: node_class - in: path - required: true - schema: - type: string - description: Node class name (e.g. "KSampler") - responses: - "200": - description: Single node definition - content: - application/json: - schema: - type: object - additionalProperties: - $ref: "#/components/schemas/NodeInfo" - "404": - description: Node class not found - - /api/embeddings: - get: - operationId: getEmbeddings - tags: [node] - summary: List available embedding names - description: Returns the list of text-encoder embeddings available on disk. - responses: - "200": - description: Embedding names - content: - application/json: - schema: - type: array - items: - type: string - - # --------------------------------------------------------------------------- - # Models - # --------------------------------------------------------------------------- - /api/models: - get: - operationId: getModelTypes - tags: [model] - summary: List model folder type names - description: Returns an array of model type names (e.g. checkpoints, loras, vae). - responses: - "200": - description: Model type names - content: - application/json: - schema: - type: array - items: - type: string - - '404': - description: "Not Found \u2014 use /api/experiment/models instead" - /api/models/{folder}: - get: - operationId: getModelsByFolder - tags: [model] - summary: List model filenames in a folder - description: Returns the names of model files in the given folder. This endpoint predates `/api/experiment/models/{folder}` and returns names only — prefer the experiment endpoint for new integrations. - parameters: - - name: folder - in: path - required: true - schema: - type: string - description: Model folder type name - responses: - "200": - description: Model filenames - content: - application/json: - schema: - type: array - items: - type: string - "404": - description: Unknown folder type - - /api/experiment/models: - get: - operationId: getModelFolders - tags: [model] - summary: List model folders with paths - description: Returns an array of model folder objects with name and folder paths. - responses: - "200": - description: Model folders - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/ModelFolder" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/experiment/models/{folder}: - get: - operationId: getModelsInFolder - tags: [model] - summary: List model files with metadata - description: Returns the model files in the given folder with richer metadata (path index, mtime, size) than the legacy `/api/models/{folder}` endpoint. - parameters: - - name: folder - in: path - required: true - schema: - type: string - description: Model folder type name - responses: - "200": - description: Model files with metadata - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/ModelFile" - "404": - description: Unknown folder type - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/experiment/models/preview/{folder}/{path_index}/{filename}: - get: - operationId: getModelPreview - tags: [model] - summary: Get model preview image - description: Returns the preview image associated with a model file, if one exists alongside the model on disk. - parameters: - - name: folder - in: path - required: true - schema: - type: string - description: Model folder type name - - name: path_index - in: path - required: true - schema: - type: integer - description: Path index within the folder - - name: filename - in: path - required: true - schema: - type: string - description: Model filename - responses: - "200": - description: Preview image (WebP) - content: - image/webp: - schema: - type: string - format: binary - "404": - description: Preview not found - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Users - # --------------------------------------------------------------------------- - /api/users: - get: - operationId: getUsersInfo - tags: [user] - summary: Get user storage info - description: | - Returns user storage configuration. In single-user mode returns - `{"storage": "server", "migrated": true/false}`. In multi-user mode - returns `{"storage": "server", "users": {"user_id": "user_dir", ...}}`. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - responses: - "200": - description: User info - content: - application/json: - schema: - type: object - properties: - storage: - type: string - description: Storage backend type (always "server") - migrated: - type: boolean - description: Whether migration from browser storage is complete (single-user) - users: - type: object - additionalProperties: - type: string - description: Map of user_id to directory name (multi-user) - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: createUser - tags: [user] - summary: Create a new user (multi-user mode) - description: Creates a new user entry. Only meaningful when ComfyUI is running in multi-user mode. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - username - properties: - username: - type: string - description: Username for the new user - responses: - "200": - description: Created user ID - content: - application/json: - schema: - type: string - description: The generated user_id - "400": - description: Username already exists or invalid - - # --------------------------------------------------------------------------- - # Userdata - # --------------------------------------------------------------------------- - /api/userdata: - get: - operationId: getUserdata - tags: [userdata] - summary: List files in a userdata directory - description: Lists files in the authenticated user's data directory. Returns either filename strings or full objects depending on the `full_info` query parameter. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: dir - in: query - required: true - schema: - type: string - description: Directory path relative to the user's data folder - - name: recurse - in: query - schema: - type: boolean - description: Recurse into subdirectories - - name: full_info - in: query - schema: - type: boolean - description: Return full file info objects instead of just names - - name: split - in: query - schema: - type: boolean - description: Split paths into directory components - responses: - "200": - description: File listing - content: - application/json: - schema: - $ref: "#/components/schemas/GetUserDataResponseFull" - "404": - description: Directory not found - - '400': - description: Bad request (e.g., invalid filename). - content: - text/plain: - schema: - type: string - '401': - description: Unauthorized. - content: - text/plain: - schema: - type: string - '500': - description: General error - content: - text/plain: - schema: - type: string - /api/v2/userdata: - get: - operationId: listUserdataV2 - tags: [userdata] - summary: List files in userdata (v2 format) - description: Lists files in the authenticated user's data directory using the v2 response shape, which always returns full objects. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: path - in: query - schema: - type: string - description: Directory path relative to user data root - responses: - "200": - description: File listing with metadata - content: - application/json: - schema: - type: array - items: - type: object - properties: - name: - type: string - path: - type: string - type: - type: string - enum: [file, directory] - size: - type: integer - modified: - type: number - description: Unix timestamp - - '404': - description: "Not Found \u2014 use /api/userdata instead" - /api/userdata/{file}: - get: - operationId: getUserdataFile - tags: [userdata] - summary: Read a userdata file - description: Reads the contents of a file from the authenticated user's data directory. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: file - in: path - required: true - schema: - type: string - description: File path relative to user data directory - responses: - "200": - description: File content - content: - application/octet-stream: - schema: - type: string - format: binary - "404": - description: File not found - '400': - description: Bad request (e.g., invalid filename). - content: - text/plain: - schema: - type: string - '401': - description: Unauthorized. - content: - text/plain: - schema: - type: string - '500': - description: General error - content: - text/plain: - schema: - type: string - post: - operationId: postUserdataFile - tags: [userdata] - summary: Write or create a userdata file - description: Writes (creates or replaces) a file in the authenticated user's data directory. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: file - in: path - required: true - schema: - type: string - description: File path relative to user data directory - - name: overwrite - in: query - schema: - type: boolean - description: Allow overwriting existing files - - name: full_info - in: query - schema: - type: boolean - description: Return full file info in response - requestBody: - required: true - content: - application/octet-stream: - schema: - type: string - format: binary - application/json: - schema: {} - responses: - "200": - description: File written - content: - application/json: - schema: - $ref: "#/components/schemas/UserDataResponseFull" - "409": - description: File exists and overwrite not set - '400': - description: Missing or invalid 'file' parameter. - content: - text/plain: - schema: - type: string - '401': - description: Unauthorized. - content: - text/plain: - schema: - type: string - '403': - description: The requested path is not allowed. - content: - text/plain: - schema: - type: string - '500': - description: General error - content: - text/plain: - schema: - type: string - delete: - operationId: deleteUserdataFile - tags: [userdata] - summary: Delete a userdata file - description: Deletes a file from the authenticated user's data directory. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: file - in: path - required: true - schema: - type: string - description: File path relative to user data directory - responses: - "204": - description: File deleted - "404": - description: File not found - - '401': - description: Unauthorized. - content: - text/plain: - schema: - type: string - '500': - description: Internal server error. - content: - text/plain: - schema: - type: string - /api/userdata/{file}/move/{dest}: - post: - operationId: moveUserdataFile - tags: [userdata] - summary: Move or rename a userdata file - description: Renames or moves a file within the authenticated user's data directory. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: file - in: path - required: true - schema: - type: string - description: Source file path - - name: dest - in: path - required: true - schema: - type: string - description: Destination file path - - name: overwrite - in: query - schema: - type: boolean - description: Allow overwriting at destination - - name: full_info - in: query - schema: - type: boolean - description: Return full file info in response - responses: - "200": - description: File moved - content: - application/json: - schema: - $ref: "#/components/schemas/UserDataResponseFull" - "404": - description: Source file not found - "409": - description: Destination exists and overwrite not set - - '400': - description: Missing or invalid parameters. - content: - text/plain: - schema: - type: string - '401': - description: Unauthorized. - content: - text/plain: - schema: - type: string - '500': - description: General error - content: - text/plain: - schema: - type: string - # --------------------------------------------------------------------------- - # Settings - # --------------------------------------------------------------------------- - /api/settings: - get: - operationId: getAllSettings - tags: [settings] - summary: Get all user settings - description: Returns all settings for the authenticated user. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - responses: - "200": - description: Settings object - content: - application/json: - schema: - type: object - additionalProperties: true - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: updateMultipleSettings - tags: [settings] - summary: Update user settings (partial merge) - description: Replaces the authenticated user's settings with the provided object. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - requestBody: - required: true - content: - application/json: - schema: - type: object - additionalProperties: true - description: Partial settings to merge - responses: - "200": - description: Settings updated - - '400': - description: Invalid request - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/settings/{id}: - get: - operationId: getSettingById - tags: [settings] - summary: Get a single setting by key - description: Returns the value of a single setting, identified by key. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: id - in: path - required: true - schema: - type: string - description: Setting key - responses: - "200": - description: Setting value (null if the setting does not exist) - content: - application/json: - schema: - nullable: true - description: The setting value (any JSON type), or null if not set - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Setting not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: updateSettingById - tags: [settings] - summary: Set a single setting value - description: Sets the value of a single setting, identified by key. - parameters: - - $ref: "#/components/parameters/ComfyUserHeader" - - name: id - in: path - required: true - schema: - type: string - description: Setting key - requestBody: - required: true - content: - application/json: - schema: - description: The setting value (any JSON type) - responses: - "200": - description: Setting updated - - '400': - description: Invalid request - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Extensions / Templates / i18n - # --------------------------------------------------------------------------- - /api/extensions: - get: - operationId: getExtensions - tags: [extensions] - summary: List frontend extension JS file paths - description: Returns the list of frontend extension JS URLs registered by custom nodes, to be loaded by the frontend on startup. - responses: - "200": - description: Array of JS file paths - content: - application/json: - schema: - type: array - items: - type: string - description: Relative path to extension JS file - - /api/workflow_templates: - get: - operationId: getWorkflowTemplates - tags: [extensions] - summary: Get workflow template mappings - description: Returns a map of custom node names to their provided workflow template names. - responses: - "200": - description: Template mappings - content: - application/json: - schema: - type: object - additionalProperties: - type: array - items: - type: string - description: Map of node pack name to array of template names - - /api/i18n: - get: - operationId: getI18n - tags: [extensions] - summary: Get internationalisation translation strings - description: Returns the URLs of translation files contributed by custom nodes, keyed by locale. - responses: - "200": - description: Translation map - content: - application/json: - schema: - type: object - additionalProperties: true - description: Nested map of locale to translation key-value pairs - - # --------------------------------------------------------------------------- - # Subgraphs - # --------------------------------------------------------------------------- - /api/global_subgraphs: - get: - operationId: getGlobalSubgraphs - tags: [subgraph] - summary: List global subgraph blueprints - description: Returns a dictionary of subgraph IDs to their metadata. - responses: - "200": - description: Subgraph metadata dictionary - content: - application/json: - schema: - type: object - additionalProperties: - $ref: "#/components/schemas/GlobalSubgraphInfo" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/global_subgraphs/{id}: - get: - operationId: getGlobalSubgraph - tags: [subgraph] - summary: Get a global subgraph with full data - description: Returns the blueprint for a globally-registered subgraph, used by the frontend to materialize the subgraph node. - parameters: - - name: id - in: path - required: true - schema: - type: string - description: Subgraph identifier - responses: - "200": - description: Full subgraph data - content: - application/json: - schema: - $ref: "#/components/schemas/GlobalSubgraphData" - "404": - description: Subgraph not found - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Node Replacements - # --------------------------------------------------------------------------- - /api/node_replacements: - get: - operationId: getNodeReplacements - tags: [node] - summary: Get node replacement mappings - description: | - Returns a dictionary mapping deprecated or replaced node class names - to their replacement node information. - responses: - "200": - description: Replacement mappings - content: - application/json: - schema: - type: object - additionalProperties: true - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Internal (x-internal: true) - # --------------------------------------------------------------------------- - /internal/logs: - get: - operationId: getInternalLogs - tags: [internal] - summary: Get server logs as text - description: Returns structured ComfyUI log entries from the in-memory log buffer. - x-internal: true - responses: - "200": - description: Log text - content: - text/plain: - schema: - type: string - - /internal/logs/raw: - get: - operationId: getInternalLogsRaw - tags: [internal] - summary: Get raw structured log entries - description: Returns the raw ComfyUI log buffer as text, together with metadata about the current size limit. - x-internal: true - responses: - "200": - description: Structured log data - content: - application/json: - schema: - type: object - properties: - entries: - type: array - items: - type: object - properties: - t: - type: number - description: Timestamp - m: - type: string - description: Message - size: - type: object - properties: - cols: - type: integer - rows: - type: integer - - /internal/logs/subscribe: - patch: - operationId: subscribeToLogs - tags: [internal] - summary: Subscribe or unsubscribe a WebSocket client to log streaming - description: Subscribes or unsubscribes the current client from live log streaming over the WebSocket. - x-internal: true - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - clientId - - enabled - properties: - clientId: - type: string - description: WebSocket client ID - enabled: - type: boolean - description: Enable or disable log streaming for this client - responses: - "200": - description: Subscription updated - - /internal/folder_paths: - get: - operationId: getInternalFolderPaths - tags: [internal] - summary: Get configured folder paths - description: Returns the filesystem paths ComfyUI is configured to load models and other assets from, keyed by folder type. - x-internal: true - responses: - "200": - description: Dictionary of folder type to paths - content: - application/json: - schema: - type: object - additionalProperties: - type: array - items: - type: array - items: - type: string - description: Map of folder type name to list of [path, ...] entries - - /internal/files/{directory_type}: - get: - operationId: getFiles - tags: [internal] - summary: List files in a directory type - description: Lists the files present in one of ComfyUI's known directories (input, output, or temp). - x-internal: true - parameters: - - name: directory_type - in: path - required: true - schema: - type: string - description: Directory type (e.g. output, input, temp) - responses: - "200": - description: Array of filenames - content: - application/json: - schema: - type: array - items: - type: string - - '400': - description: Invalid directory type - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Assets (x-feature-gate: enable-assets) - # --------------------------------------------------------------------------- - /api/assets/hash/{hash}: - head: - operationId: checkAssetByHash - tags: [assets] - summary: Check if an asset with the given hash exists - description: Returns 204 if an asset with the given content hash already exists, 404 otherwise. Used by clients to deduplicate uploads before transferring bytes. - x-feature-gate: enable-assets - parameters: - - name: hash - in: path - required: true - schema: - type: string - description: "Blake3 hash of the asset (e.g. blake3:abc123...)" - responses: - "200": - description: Asset exists - "404": - description: No asset with this hash - - '400': - description: Invalid hash format - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets: - get: - operationId: listAssets - tags: [assets] - summary: List assets with filtering and pagination - description: Returns a paginated list of assets, optionally filtered by tags, name, or other query parameters. - x-feature-gate: enable-assets - parameters: - - name: limit - in: query - schema: - type: integer - default: 50 - - name: offset - in: query - schema: - type: integer - default: 0 - - name: include_tags - in: query - schema: - type: array - items: - type: string - style: form - explode: true - description: Tags that assets must have (AND logic) - - name: exclude_tags - in: query - schema: - type: array - items: - type: string - style: form - explode: true - description: Tags that assets must not have - - name: name_contains - in: query - schema: - type: string - description: Filter assets whose name contains this substring - - name: metadata_filter - in: query - schema: - type: string - description: JSON-encoded metadata key/value filter - - name: sort - in: query - schema: - type: string - description: Field to sort by - - name: order - in: query - schema: - type: string - enum: [asc, desc] - description: Sort direction - - name: include_public - in: query - schema: - type: boolean - x-runtime: [cloud] - description: "[cloud-only] Include workspace-public assets in addition to the caller's own." - - name: asset_hash - in: query - schema: - type: string - x-runtime: [cloud] - description: "[cloud-only] Filter by exact content hash." - responses: - "200": - description: Asset list - content: - application/json: - schema: - $ref: "#/components/schemas/ListAssetsResponse" - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: uploadAsset - tags: [assets] - summary: Upload a new asset - description: Uploads a new asset (binary content plus metadata) and registers it in the asset database. - x-feature-gate: enable-assets - requestBody: - required: true - content: - multipart/form-data: - schema: - type: object - required: - - file - properties: - file: - type: string - format: binary - description: Asset file to upload - name: - type: string - description: Display name for the asset - tags: - type: string - description: Comma-separated tags - user_metadata: - type: string - description: JSON-encoded user metadata - hash: - type: string - description: "Blake3 hash of the file content (e.g. blake3:abc123...)" - mime_type: - type: string - description: MIME type of the file (overrides auto-detected type) - preview_id: - type: string - format: uuid - description: ID of an existing asset to use as the preview image - id: - type: string - format: uuid - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Client-supplied asset ID for idempotent creation. If an asset with this ID already exists, the existing asset is returned." - application/json: - schema: - type: object - x-runtime: [cloud] - description: "[cloud-only] URL-based asset upload. Caller supplies a URL instead of a file body; the server fetches the content." - required: - - url - properties: - url: - type: string - format: uri - description: "[cloud-only] URL of the file to import as an asset" - name: - type: string - description: Display name for the asset - tags: - type: string - description: Comma-separated tags - user_metadata: - type: string - description: JSON-encoded user metadata - hash: - type: string - description: "Blake3 hash of the file content (e.g. blake3:abc123...)" - mime_type: - type: string - description: MIME type of the file (overrides auto-detected type) - preview_id: - type: string - format: uuid - description: ID of an existing asset to use as the preview image - id: - type: string - format: uuid - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Client-supplied asset ID for idempotent creation. If an asset with this ID already exists, the existing asset is returned." - responses: - "201": - description: Asset created - content: - application/json: - schema: - $ref: "#/components/schemas/AssetCreated" - - '200': - description: Asset already exists (returned existing asset) - content: - application/json: - schema: - $ref: '#/components/schemas/AssetCreated' - '400': - description: Invalid request (bad file, invalid URL, invalid content type, etc.) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '403': - description: Source URL requires authentication or access denied - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Source URL not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '413': - description: File too large - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '415': - description: Unsupported media type - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Download failed due to network error or timeout - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/from-hash: - post: - operationId: createAssetFromHash - tags: [assets] - summary: Create an asset reference from an existing hash - description: Registers a new asset that references existing content by hash, without re-uploading the bytes. - x-feature-gate: enable-assets - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - hash - - name - properties: - hash: - type: string - description: Blake3 hash of existing content - name: - type: string - description: Display name - tags: - type: array - items: - type: string - user_metadata: - type: object - additionalProperties: true - mime_type: - type: string - nullable: true - x-runtime: [cloud] - description: "[cloud-only] MIME type of the content, so the type is preserved without re-inspecting content. Ignored by local ComfyUI." - responses: - "201": - description: Asset created from hash - content: - application/json: - schema: - $ref: "#/components/schemas/AssetCreated" - - '200': - description: Asset reference already exists (returned existing) - content: - application/json: - schema: - $ref: '#/components/schemas/AssetCreated' - '400': - description: Invalid request (bad hash format, invalid tags, etc.) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Source asset with given hash not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/{id}: - get: - operationId: getAssetById - tags: [assets] - summary: Get asset metadata - description: Returns the metadata for a single asset. - x-feature-gate: enable-assets - parameters: - - name: id - in: path - description: The asset ID. - required: true - schema: - type: string - format: uuid - responses: - "200": - description: Asset metadata - content: - application/json: - schema: - $ref: "#/components/schemas/Asset" - "404": - description: Asset not found - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - put: - operationId: updateAsset - tags: [assets] - summary: Update asset metadata - description: Updates the mutable metadata of an asset (name, tags, etc.). Binary content is immutable. - x-feature-gate: enable-assets - parameters: - - name: id - in: path - description: The asset ID. - required: true - schema: - type: string - format: uuid - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - name: - type: string - description: New display name for the asset - user_metadata: - type: object - additionalProperties: true - description: Custom user metadata to set - preview_id: - type: string - format: uuid - description: ID of the asset to use as the preview - mime_type: - type: string - nullable: true - x-runtime: [cloud] - description: "[cloud-only] MIME type override when auto-detection was wrong. Ignored by local ComfyUI." - responses: - "200": - description: Asset updated - content: - application/json: - schema: - $ref: "#/components/schemas/AssetUpdated" - '400': - description: Invalid request (no fields provided) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Asset not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: deleteAsset - tags: [assets] - summary: Delete an asset - description: Removes an asset entry. Depending on the server configuration, the underlying content may also be deleted. - x-feature-gate: enable-assets - parameters: - - name: id - in: path - description: The asset ID. - required: true - schema: - type: string - format: uuid - - name: delete_content - in: query - schema: - type: boolean - description: Also delete the underlying content file - responses: - "204": - description: Asset deleted - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Asset not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '409': - description: Asset cannot be deleted because it is referenced by another resource (e.g., workflow version) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/{id}/content: - get: - operationId: getAssetContent - tags: [assets] - summary: Download asset file content - description: Returns the binary content of an asset. Supports range requests. - x-feature-gate: enable-assets - parameters: - - name: id - in: path - description: The asset ID. - required: true - schema: - type: string - format: uuid - responses: - "200": - description: Asset file content - content: - application/octet-stream: - schema: - type: string - format: binary - "404": - description: Asset not found - - /api/assets/{id}/tags: - post: - operationId: addAssetTags - tags: [assets] - summary: Add tags to an asset - description: Adds one or more tags to an asset. - x-feature-gate: enable-assets - parameters: - - name: id - in: path - description: The asset ID. - required: true - schema: - type: string - format: uuid - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - tags - properties: - tags: - type: array - items: - type: string - responses: - "200": - description: Tags added - content: - application/json: - schema: - $ref: "#/components/schemas/TagsModificationResponse" - '400': - description: Invalid request - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Asset not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error (e.g., reserved tag) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: removeAssetTags - tags: [assets] - summary: Remove tags from an asset - description: Removes one or more tags from an asset. - x-feature-gate: enable-assets - parameters: - - name: id - in: path - description: The asset ID. - required: true - schema: - type: string - format: uuid - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - tags - properties: - tags: - type: array - items: - type: string - responses: - "200": - description: Tags removed - content: - application/json: - schema: - $ref: "#/components/schemas/TagsModificationResponse" - - '400': - description: Invalid request - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Asset not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error (e.g., reserved tag) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/tags: - get: - operationId: listTags - tags: [assets] - summary: List all known tags with counts - description: Returns the list of all tags known to the asset database, with counts. - x-feature-gate: enable-assets - parameters: - - name: limit - in: query - schema: - type: integer - - name: offset - in: query - schema: - type: integer - - name: search - in: query - schema: - type: string - description: Search term for tag name - responses: - "200": - description: Tag list - content: - application/json: - schema: - $ref: "#/components/schemas/ListTagsResponse" - - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/tags/refine: - get: - operationId: getAssetTagHistogram - tags: [assets] - summary: Get tag counts for assets matching current filters - description: Returns suggested additional tags that would refine a filtered asset query, together with the count of assets each tag would select. - x-feature-gate: enable-assets - parameters: - - name: include_tags - in: query - schema: - type: array - items: - type: string - style: form - explode: true - description: Tags that assets must have (AND logic) - - name: exclude_tags - in: query - schema: - type: array - items: - type: string - style: form - explode: true - description: Tags that assets must not have - - name: name_contains - in: query - schema: - type: string - description: Filter assets whose name contains this substring - - name: metadata_filter - in: query - schema: - type: string - description: JSON-encoded metadata key/value filter - - name: limit - in: query - schema: - type: integer - - name: offset - in: query - schema: - type: integer - - name: sort - in: query - schema: - type: string - description: Field to sort by - - name: order - in: query - schema: - type: string - enum: [asc, desc] - description: Sort direction - responses: - "200": - description: Tag histogram - content: - application/json: - schema: - $ref: "#/components/schemas/AssetTagHistogramResponse" - - '400': - description: Invalid request parameters - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/seed: - post: - operationId: seedAssets - tags: [assets] - summary: Trigger asset scan/seed from filesystem - description: Starts a background job that scans the configured directories and registers any assets not yet present in the asset database. - x-feature-gate: enable-assets - requestBody: - required: false - content: - application/json: - schema: - type: object - properties: - roots: - type: array - items: - type: string - description: Root folder paths to scan (if omitted, scans all) - responses: - "200": - description: Seed started - content: - application/json: - schema: - type: object - properties: - status: - type: string - - /api/assets/seed/status: - get: - operationId: getAssetSeedStatus - tags: [assets] - summary: Get asset scan progress - description: Returns the progress and status of the most recently-started asset seed job. - x-feature-gate: enable-assets - responses: - "200": - description: Scan progress - content: - application/json: - schema: - type: object - additionalProperties: true - description: Scan progress details (files scanned, total, status, etc.) - - /api/assets/seed/cancel: - post: - operationId: cancelAssetSeed - tags: [assets] - summary: Cancel an in-progress asset scan - description: Requests cancellation of the currently-running asset seed job. - x-feature-gate: enable-assets - responses: - "200": - description: Scan cancelled - content: - application/json: - schema: - type: object - properties: - status: - type: string - - /api/assets/prune: - post: - operationId: pruneAssets - tags: [assets] - summary: Mark assets whose backing files no longer exist on disk - description: Starts a background job that removes asset entries whose underlying content no longer exists on disk. - x-feature-gate: enable-assets - responses: - "200": - description: Prune result - content: - application/json: - schema: - type: object - properties: - status: - type: string - marked: - type: integer - description: Number of assets marked as missing - - # =========================================================================== - # Cloud-runtime FE-facing operations - # - # These operations are served by the cloud runtime. The local runtime returns - # 404 for all of these paths. Each operation is tagged x-runtime: [cloud]. - # =========================================================================== - - # --------------------------------------------------------------------------- - # Jobs / prompts (cloud) - # --------------------------------------------------------------------------- - /api/jobs/{job_id}/cancel: - post: - operationId: cancelJob - tags: [queue] - summary: Cancel a running or pending job - description: "[cloud-only] Requests cancellation of a job. If the job is currently executing, execution is interrupted. If it is pending in the queue, it is removed." - x-runtime: [cloud] - parameters: - - name: job_id - in: path - required: true - schema: - type: string - format: uuid - description: The job ID to cancel. - responses: - "200": - description: Cancellation accepted - content: - application/json: - schema: - $ref: "#/components/schemas/JobCancelResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: Bad Request - job_id is not a valid UUID (emitted by request validation before the handler runs) - content: - application/json: - schema: - $ref: '#/components/schemas/BindingErrorResponse' - '500': - description: Internal server error - cancellation failed - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/job/{job_id}/status: - get: - operationId: getJobStatus - tags: [queue] - summary: Get status of a cloud job - deprecated: true - description: | - **Deprecated.** This endpoint is superseded by `GET /api/jobs/{job_id}`. - Clients should migrate; the endpoint is retained for backward - compatibility but will be removed in a future release. - x-runtime: [cloud] - parameters: - - name: job_id - in: path - required: true - schema: - type: string - format: uuid - description: The job ID to check status for. - responses: - "200": - description: Job status - content: - application/json: - schema: - $ref: "#/components/schemas/JobStatusResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '403': - description: Forbidden - job belongs to another user - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/prompt/{prompt_id}: - get: - operationId: getCloudPrompt - tags: [prompt] - summary: Get a cloud prompt by ID - description: "[cloud-only] Returns the full prompt record for a cloud-executed prompt, including the submitted workflow graph and execution metadata." - x-runtime: [cloud] - parameters: - - name: prompt_id - in: path - required: true - schema: - type: string - format: uuid - description: The prompt ID to fetch. - responses: - "200": - description: Cloud prompt detail - content: - application/json: - schema: - $ref: "#/components/schemas/CloudPrompt" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/history_v2: - get: - operationId: getHistory - tags: [history] - summary: Get paginated execution history (v2) - deprecated: true - description: | - **Deprecated.** This endpoint is superseded by `GET /api/jobs`. - Clients should migrate; the endpoint is retained for backward - compatibility but will be removed in a future release. - x-runtime: [cloud] - parameters: - - name: limit - in: query - schema: - type: integer - default: 20 - description: Maximum number of results - - name: offset - in: query - schema: - type: integer - default: 0 - description: Pagination offset - - name: status - in: query - schema: - type: string - description: Filter by execution status - responses: - "200": - description: History list - content: - application/json: - schema: - $ref: "#/components/schemas/HistoryResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/history_v2/{prompt_id}: - get: - operationId: getHistoryForPrompt - tags: [history] - summary: Get v2 history for a specific prompt - deprecated: true - description: | - **Deprecated.** This endpoint is superseded by `GET /api/jobs/{prompt_id}`. - Clients should migrate; the endpoint is retained for backward - compatibility but will be removed in a future release. - x-runtime: [cloud] - parameters: - - name: prompt_id - in: path - required: true - schema: - type: string - format: uuid - description: The prompt ID to fetch history for. - responses: - "200": - description: History entry - content: - application/json: - schema: - $ref: "#/components/schemas/HistoryDetailResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/logs: - get: - operationId: getLogs - tags: [system] - summary: Get cloud execution logs - deprecated: true - description: | - **Deprecated.** This endpoint returns a static placeholder response and - provides no real log data. It is retained only to avoid breaking clients - that still call it. Clients should remove their dependency; the endpoint - will be removed in a future release. - x-runtime: [cloud] - parameters: - - name: job_id - in: query - schema: - type: string - description: Filter logs by job ID - - name: limit - in: query - schema: - type: integer - default: 100 - description: Maximum number of log entries - - name: offset - in: query - schema: - type: integer - default: 0 - description: Pagination offset - responses: - "200": - description: Log entries - content: - application/json: - schema: - $ref: "#/components/schemas/LogsResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - # --------------------------------------------------------------------------- - # Assets extensions (cloud) - # --------------------------------------------------------------------------- - /api/assets/download: - post: - operationId: createAssetDownload - tags: [assets] - summary: Download assets to cloud runtime - description: "[cloud-only] Initiates a download of one or more assets to the cloud runtime environment. Returns a task ID for tracking download progress via WebSocket." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - assets - properties: - assets: - type: array - items: - $ref: "#/components/schemas/AssetDownloadRequest" - description: Assets to download - responses: - "202": - description: Download task accepted - content: - application/json: - schema: - type: object - required: - - task_id - - status - properties: - task_id: - type: string - format: uuid - description: ID of the download task; use to poll status. - status: - type: string - enum: [created, running, completed, failed] - description: Current task status (typically `created` on initial creation). - message: - type: string - description: Human-readable task message. - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '200': - description: File already exists in storage - asset created/returned immediately - content: - application/json: - schema: - $ref: '#/components/schemas/AssetCreated' - '422': - description: Validation errors - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/export: - post: - operationId: createAssetExport - tags: [assets] - summary: Export assets as a downloadable archive - description: "[cloud-only] Initiates a bulk export of assets. Returns a task ID for tracking progress via WebSocket. When complete, the export can be downloaded via the exports endpoint." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - job_ids: - type: array - items: - type: string - description: Job IDs whose associated assets should all be included in the ZIP bundle. - asset_ids: - type: array - items: - type: string - format: uuid - description: Asset IDs to include in the ZIP bundle. Additive to assets associated with provided job IDs. - export_name: - type: string - description: Name for the export archive - naming_strategy: - type: string - enum: [group_by_job_id, preserve, asset_id, group_by_job_time] - default: group_by_job_time - description: "Strategy for naming files in the ZIP: group by job ID, preserve original names, use the asset ID, or group by job creation time." - job_asset_name_filters: - type: object - additionalProperties: - type: array - minItems: 1 - items: - type: string - description: Optional per-job asset name filters. When provided for a job ID, only assets whose name matches one of the listed names are included. - responses: - "202": - description: Export task accepted - content: - application/json: - schema: - type: object - required: - - task_id - - status - properties: - task_id: - type: string - format: uuid - description: ID of the export task; use to poll status. - status: - type: string - enum: [created, running, completed, failed] - description: Current task status (typically `created` on initial creation). - message: - type: string - description: Human-readable task message. - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/exports/{exportName}: - get: - operationId: getAssetExport - tags: [assets] - summary: Download a completed asset export - description: "[cloud-only] Returns the archive file for a completed asset export." - x-runtime: [cloud] - parameters: - - name: exportName - in: path - required: true - schema: - type: string - description: Name of the export to download - responses: - "200": - description: Export archive file - content: - application/zip: - schema: - type: string - format: binary - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: Invalid export name - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/from-workflow: - post: - operationId: postAssetsFromWorkflow - tags: [assets] - summary: Create asset records from a workflow execution - description: "[cloud-only] Registers output files from a workflow execution as assets in the asset database." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - prompt_id - properties: - prompt_id: - type: string - format: uuid - description: Prompt ID whose outputs should be registered as assets - tags: - type: array - items: - type: string - description: Tags to apply to the created assets - responses: - "200": - description: Assets created or referenced - content: - application/json: - schema: - type: object - properties: - assets: - type: array - items: - $ref: "#/components/schemas/Asset" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/import: - post: - operationId: importPublishedAssets - tags: [assets] - summary: "[cloud-only] Import published assets into the caller's library" - description: | - [cloud-only] Imports the specified published assets into the caller's asset library. New DB records reference the same storage objects; no file copying occurs. Assets the caller already owns (by hash) are deduplicated. The `id` field on each returned `AssetInfo` is the caller's newly-created private asset ID, not the published asset ID supplied in the request. - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ImportPublishedAssetsRequest" - responses: - "200": - description: Successfully imported assets - content: - application/json: - schema: - $ref: "#/components/schemas/ImportPublishedAssetsResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/assets/remote-metadata: - get: - operationId: getRemoteAssetMetadata - tags: [assets] - summary: Fetch metadata for a remote asset URL - description: "[cloud-only] Fetches and returns metadata (content type, size, filename) for a remote URL without downloading the full content." - x-runtime: [cloud] - parameters: - - name: url - in: query - required: true - schema: - type: string - format: uri - description: URL to inspect - responses: - "200": - description: Remote metadata - content: - application/json: - schema: - $ref: "#/components/schemas/AssetMetadataResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '422': - description: Failed to retrieve metadata from source - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Custom nodes / hub (cloud) - # --------------------------------------------------------------------------- - /api/experiment/nodes: - get: - operationId: getNodeInfoSchema - tags: [runtime-only] - summary: Get pre-rendered node info schema - description: "[cloud-only] Returns the static ComfyUI object_info schema, identical for every caller, rendered once at startup with empty model/user-file context. Served by a raw HTTP handler that writes pre-rendered bytes with ETag + Cache-Control validators for RFC 7232 conditional GETs." - x-runtime: [cloud] - parameters: - - name: If-None-Match - in: header - required: false - schema: - type: string - description: Entity tag previously returned by this endpoint. When present and matching, the server returns 304 Not Modified. - responses: - "200": - description: Node info schema - headers: - ETag: - schema: - type: string - description: Entity tag for conditional request validation - Cache-Control: - schema: - type: string - description: Cache directives for the response - content: - application/json: - schema: - type: object - additionalProperties: - $ref: "#/components/schemas/NodeInfo" - "304": - description: Not Modified — returned when the client sends a matching If-None-Match header - post: - operationId: installCloudNode - tags: [node] - summary: Install a custom node package - description: "[cloud-only] Installs a custom node package in the cloud runtime by ID or repository URL." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - id - properties: - id: - type: string - description: Node package ID or repository URL - version: - type: string - description: Specific version to install - responses: - "200": - description: Node installed - content: - application/json: - schema: - $ref: "#/components/schemas/CloudNode" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/experiment/nodes/{id}: - get: - operationId: getNodeByID - tags: [runtime-only] - summary: Get a single node definition by ID - description: "[cloud-only] Returns one node's definition from the pre-indexed object_info schema. Served by a raw HTTP handler that writes pre-rendered bytes with ETag + Cache-Control validators for RFC 7232 conditional GETs." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: Node class identifier - - name: If-None-Match - in: header - required: false - schema: - type: string - description: Entity tag previously returned by this endpoint. When present and matching, the server returns 304 Not Modified. - responses: - "200": - description: Single node definition - headers: - ETag: - schema: - type: string - description: Entity tag for conditional request validation - Cache-Control: - schema: - type: string - description: Cache directives for the response - content: - application/json: - schema: - $ref: "#/components/schemas/NodeInfo" - "304": - description: Not Modified — returned when the client sends a matching If-None-Match header - "404": - description: Node not found - delete: - operationId: uninstallCloudNode - tags: [node] - summary: Uninstall a custom node package - description: "[cloud-only] Removes a custom node package from the cloud runtime." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: Custom node package ID - responses: - "204": - description: Node uninstalled - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/hub/assets/upload-url: - post: - operationId: createHubAssetUploadUrl - tags: [hub] - summary: Get a pre-signed upload URL for a hub asset - description: "[cloud-only] Returns a pre-signed URL that can be used to upload an asset file directly to storage." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - filename - - content_type - properties: - filename: - type: string - description: Name of the file to upload - content_type: - type: string - description: MIME type of the file - size: - type: integer - format: int64 - description: File size in bytes - responses: - "200": - description: Upload URL - content: - application/json: - schema: - type: object - properties: - upload_url: - type: string - format: uri - description: Pre-signed upload URL - asset_url: - type: string - format: uri - description: Public URL after upload completes - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/labels: - get: - operationId: listHubLabels - tags: [hub] - summary: List available hub labels - description: "[cloud-only] Returns the list of labels/categories available for tagging hub content." - x-runtime: [cloud] - responses: - "200": - description: Label list - content: - application/json: - schema: - $ref: "#/components/schemas/HubLabelListResponse" - '400': - description: Bad request (e.g. invalid type parameter) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/profiles: - get: - operationId: listHubProfiles - tags: [hub] - summary: List hub user profiles - description: "[cloud-only] Returns a paginated list of public hub user profiles." - x-runtime: [cloud] - parameters: - - name: limit - in: query - schema: - type: integer - description: Maximum number of results - - name: offset - in: query - schema: - type: integer - description: Pagination offset - - name: search - in: query - schema: - type: string - description: Search by username or display name - responses: - "200": - description: Profile list - content: - application/json: - schema: - type: object - properties: - profiles: - type: array - items: - $ref: "#/components/schemas/HubProfile" - total: - type: integer - has_more: - type: boolean - post: - operationId: createHubProfile - tags: [hub] - summary: Create a Hub profile - description: "[cloud-only] Creates a hub profile for the specified workspace. Username is immutable after creation." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/CreateHubProfileRequest" - responses: - "201": - description: Hub profile created - content: - application/json: - schema: - $ref: "#/components/schemas/HubProfile" - "400": - description: Bad request (e.g. invalid username) - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "409": - description: Username already taken or profile already exists - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/profiles/{username}: - get: - operationId: getHubProfile - tags: [hub] - summary: Get a hub profile by username - description: "[cloud-only] Returns the public hub profile for the given username." - x-runtime: [cloud] - parameters: - - name: username - in: path - required: true - schema: - type: string - description: Hub username - responses: - "200": - description: Profile - content: - application/json: - schema: - $ref: "#/components/schemas/HubProfile" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/profiles/check: - get: - operationId: checkHubUsername - tags: [hub] - summary: Check if a hub username is available - description: "[cloud-only] Returns whether the given username is available for registration." - x-runtime: [cloud] - parameters: - - name: username - in: query - required: true - schema: - type: string - description: Username to check - responses: - "200": - description: Availability result - content: - application/json: - schema: - type: object - properties: - available: - type: boolean - username: - type: string - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/profiles/me: - get: - operationId: getMyHubProfile - tags: [hub] - summary: Get the authenticated user's hub profile - description: "[cloud-only] Returns the hub profile of the currently authenticated user." - x-runtime: [cloud] - responses: - "200": - description: Profile - content: - application/json: - schema: - $ref: "#/components/schemas/HubProfile" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '404': - description: No hub profile exists - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - put: - operationId: updateMyHubProfile - tags: [hub] - summary: Update the authenticated user's hub profile - description: "[cloud-only] Updates the hub profile of the currently authenticated user." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - username: - type: string - display_name: - type: string - bio: - type: string - avatar_url: - type: string - format: uri - links: - type: array - items: - type: string - format: uri - responses: - "200": - description: Updated profile - content: - application/json: - schema: - $ref: "#/components/schemas/HubProfile" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "409": - description: Conflict - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/hub/workflows: - get: - operationId: listHubWorkflows - tags: [hub] - summary: List published hub workflows - description: "[cloud-only] Returns a paginated list of publicly shared workflows on the hub." - x-runtime: [cloud] - parameters: - - name: limit - in: query - schema: - type: integer - description: Maximum number of results - - name: offset - in: query - schema: - type: integer - description: Pagination offset - - name: sort - in: query - schema: - type: string - description: Sort field (e.g. created_at, likes) - - name: order - in: query - schema: - type: string - enum: [asc, desc] - description: Sort direction - - name: search - in: query - schema: - type: string - description: Search by title or description - - name: labels - in: query - schema: - type: string - description: Filter by label IDs (comma-separated) - responses: - "200": - description: Hub workflow list - content: - application/json: - schema: - $ref: "#/components/schemas/HubWorkflowListResponse" - '400': - description: Bad request (e.g. malformed pagination cursor) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: Profile not found (when filtering by username) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: publishHubWorkflow - tags: [hub] - summary: Publish a workflow to the hub - description: "[cloud-only] Publishes a workflow to the hub with metadata, thumbnail, and sample images." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/PublishHubWorkflowRequest" - responses: - "200": - description: Workflow published to hub - content: - application/json: - schema: - $ref: "#/components/schemas/HubWorkflowDetail" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Workflow or profile not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/workflows/{share_id}: - get: - operationId: getHubWorkflow - tags: [hub] - summary: Get a published hub workflow by share ID - description: "[cloud-only] Returns the full details of a published workflow on the hub." - x-runtime: [cloud] - parameters: - - name: share_id - in: path - required: true - schema: - type: string - description: Workflow share ID - responses: - "200": - description: Hub workflow - content: - application/json: - schema: - $ref: "#/components/schemas/HubWorkflowDetail" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '413': - description: Workflow JSON too large - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: deleteHubWorkflow - tags: [hub] - summary: Unpublish a workflow from the hub - description: "[cloud-only] Removes a workflow from the hub listing." - x-runtime: [cloud] - parameters: - - name: share_id - in: path - required: true - schema: - type: string - description: Workflow share ID - responses: - "204": - description: Successfully unpublished - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Workflow not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/hub/workflows/index: - get: - operationId: listHubWorkflowIndex - tags: [hub] - summary: Get the hub workflow index - description: "[cloud-only] Returns the lightweight index of all hub workflows for client-side search and navigation." - x-runtime: [cloud] - responses: - "200": - description: Workflow index - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/HubWorkflowIndexEntry" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Workflows (cloud) - # --------------------------------------------------------------------------- - /api/workflows: - get: - operationId: listWorkflows - tags: [workflows] - summary: List cloud workflows - description: "[cloud-only] Returns a paginated list of the authenticated user's cloud workflows." - x-runtime: [cloud] - parameters: - - name: limit - in: query - schema: - type: integer - description: Maximum number of results - - name: offset - in: query - schema: - type: integer - description: Pagination offset - - name: sort - in: query - schema: - type: string - description: Sort field - - name: order - in: query - schema: - type: string - enum: [asc, desc] - description: Sort direction - - name: search - in: query - schema: - type: string - description: Search by workflow name - responses: - "200": - description: Workflow list - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowListResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: createWorkflow - tags: [workflows] - summary: Create a new cloud workflow - description: "[cloud-only] Creates a new cloud workflow with the provided name and optional initial content." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - name - properties: - name: - type: string - description: Workflow name - description: - type: string - description: Workflow description - content: - type: object - additionalProperties: true - description: Initial workflow graph JSON - responses: - "201": - description: Workflow created - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workflows/{workflow_id}: - get: - operationId: getWorkflow - tags: [workflows] - summary: Get a cloud workflow by ID - description: "[cloud-only] Returns the metadata for a cloud workflow." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - responses: - "200": - description: Workflow detail - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '403': - description: Forbidden - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - patch: - operationId: updateWorkflow - tags: [workflows] - summary: Update a cloud workflow - description: "[cloud-only] Updates the metadata (name, description) of an existing cloud workflow." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - name: - type: string - description: - type: string - responses: - "200": - description: Workflow updated - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: deleteWorkflow - tags: [workflows] - summary: Delete a cloud workflow - description: "[cloud-only] Deletes a cloud workflow and all its versions." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - responses: - "204": - description: Workflow deleted - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workflows/{workflow_id}/content: - get: - operationId: getWorkflowContent - tags: [workflows] - summary: Get the content of a cloud workflow - description: "[cloud-only] Returns the full workflow graph JSON for the latest version of a cloud workflow." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - - name: version_id - in: query - schema: - type: string - description: Specific version ID to fetch - responses: - "200": - description: Workflow content - content: - application/json: - schema: - type: object - additionalProperties: true - description: The full workflow graph JSON - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '403': - description: Forbidden - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - put: - operationId: updateCloudWorkflowContent - tags: [workflows] - summary: Update the content of a cloud workflow - description: "[cloud-only] Saves new workflow graph JSON as a new version of the cloud workflow." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - requestBody: - required: true - content: - application/json: - schema: - type: object - additionalProperties: true - description: The workflow graph JSON to save - responses: - "200": - description: Content updated - content: - application/json: - schema: - $ref: "#/components/schemas/CloudWorkflowVersion" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/workflows/{workflow_id}/fork: - post: - operationId: forkWorkflow - tags: [workflows] - summary: Fork a cloud workflow - description: "[cloud-only] Creates a copy of a cloud workflow under the authenticated user's account." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID to fork. - requestBody: - required: false - content: - application/json: - schema: - type: object - properties: - name: - type: string - description: Name for the forked workflow (defaults to original name) - responses: - "201": - description: Forked workflow - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '403': - description: Forbidden - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workflows/{workflow_id}/versions: - get: - operationId: listCloudWorkflowVersions - tags: [workflows] - summary: List versions of a cloud workflow - description: "[cloud-only] Returns the version history of a cloud workflow." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - - name: limit - in: query - schema: - type: integer - description: Maximum number of results - - name: offset - in: query - schema: - type: integer - description: Pagination offset - responses: - "200": - description: Version list - content: - application/json: - schema: - type: object - properties: - versions: - type: array - items: - $ref: "#/components/schemas/CloudWorkflowVersion" - total: - type: integer - has_more: - type: boolean - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - post: - operationId: createWorkflowVersion - tags: [workflows] - summary: Create a new cloud workflow version - description: "[cloud-only] Creates a new workflow version with updated workflow JSON. Uses optimistic concurrency via base_version." - x-runtime: [cloud] - parameters: - - name: workflow_id - in: path - required: true - schema: - type: string - format: uuid - description: The workflow ID. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/CreateWorkflowVersionRequest" - responses: - "201": - description: Version created - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowVersionResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden — not the workflow owner - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "409": - description: Version conflict — base_version does not match latest - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workflows/published/{share_id}: - get: - operationId: getPublishedWorkflow - tags: [workflows] - summary: Get a published workflow by share ID - description: "[cloud-only] Returns a publicly published cloud workflow by its share identifier." - x-runtime: [cloud] - parameters: - - name: share_id - in: path - required: true - schema: - type: string - description: The workflow share ID. - responses: - "200": - description: Published workflow - content: - application/json: - schema: - $ref: "#/components/schemas/PublishedWorkflowDetail" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '413': - description: Workflow JSON too large - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Auth / session (cloud) - # --------------------------------------------------------------------------- - /api/auth/session: - get: - operationId: getAuthSession - tags: [auth] - summary: Get the current authentication session - description: "[cloud-only] Returns the current session state for the authenticated user, including user identity and active workspace." - x-runtime: [cloud] - responses: - "200": - description: Session info - content: - application/json: - schema: - $ref: "#/components/schemas/AuthSession" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - post: - operationId: createSession - tags: [auth] - summary: Create a session cookie - description: "[cloud-only] Creates a session cookie from the bearer token in the Authorization header. Returns a Set-Cookie header with a secure HttpOnly session cookie. Cookie authentication is not allowed for this endpoint." - x-runtime: [cloud] - responses: - "200": - description: Session created - content: - application/json: - schema: - $ref: "#/components/schemas/CreateSessionResponse" - "400": - description: Bad request — invalid or expired ID token - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: deleteSession - tags: [auth] - summary: Delete session cookie (logout) - description: "[cloud-only] Clears the session cookie and optionally revokes the session on the server." - x-runtime: [cloud] - responses: - "200": - description: Session deleted - content: - application/json: - schema: - $ref: "#/components/schemas/DeleteSessionResponse" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/auth/token: - post: - operationId: exchangeToken - tags: [auth] - summary: Exchange credentials for an access token - description: "[cloud-only] Exchanges authentication credentials (e.g. an authorization code) for an access token." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - grant_type - properties: - grant_type: - type: string - enum: [authorization_code, refresh_token] - description: OAuth2 grant type - code: - type: string - description: Authorization code (for authorization_code grant) - refresh_token: - type: string - description: Refresh token (for refresh_token grant) - redirect_uri: - type: string - format: uri - description: Redirect URI used in the authorization request - responses: - "200": - description: Token response - content: - application/json: - schema: - $ref: "#/components/schemas/ExchangeTokenResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Workspace not found or user not a member - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /.well-known/jwks.json: - get: - operationId: getJwks - tags: [auth] - summary: Get JSON Web Key Set - description: "[cloud-only] Returns the JSON Web Key Set (JWKS) used to verify JWTs issued by the cloud authentication service." - x-runtime: [cloud] - responses: - "200": - description: JWKS - content: - application/json: - schema: - $ref: "#/components/schemas/JwksResponse" - - # --------------------------------------------------------------------------- - # OAuth 2.1 / RFC 7591 Dynamic Client Registration (cloud) - # --------------------------------------------------------------------------- - /.well-known/oauth-authorization-server: - get: - operationId: getOAuthAuthorizationServer - tags: [auth] - summary: "[cloud-only] OAuth 2.1 authorization-server metadata (RFC 8414)" - description: "[cloud-only] Public metadata document for OAuth 2.1 clients. Cached 5 minutes." - x-runtime: [cloud] - security: [] - responses: - "200": - description: Authorization-server metadata - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthAuthorizationServerMetadata" - "404": - description: OAuth disabled - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /.well-known/oauth-protected-resource: - get: - operationId: getOAuthProtectedResource - tags: [auth] - summary: "[cloud-only] OAuth 2.1 protected-resource metadata (RFC 9728)" - description: "[cloud-only] Public metadata describing the currently advertised protected resource. Cached 5 minutes." - x-runtime: [cloud] - security: [] - responses: - "200": - description: Protected-resource metadata - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthProtectedResourceMetadata" - "404": - description: OAuth disabled or no active resource configured - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /oauth/authorize: - get: - operationId: getOAuthAuthorize - tags: [auth] - summary: "[cloud-only] Begin or resume an OAuth 2.1 authorization request" - description: | - [cloud-only] Two modes: - - **Initial entry** (OAuth params present): validates client/redirect/resource/scopes, persists a server-side authorization-request row, and either redirects (no session / unverified email) to the configured frontend login URL carrying only the opaque `oauth_request_id`, or returns the JSON consent challenge for the frontend to render. - - **Resume** (`oauth_request_id` present): loads the server-side row, fails closed if expired/consumed/unknown, returns the JSON consent challenge. Browser-replayed OAuth params are intentionally ignored. - - The frontend renders the consent UI from the JSON payload and POSTs the user's decision back to this endpoint. - x-runtime: [cloud] - security: [] - parameters: - - { name: response_type, in: query, required: false, schema: { type: string } } - - { name: client_id, in: query, required: false, schema: { type: string } } - - { name: redirect_uri, in: query, required: false, schema: { type: string } } - - { name: scope, in: query, required: false, schema: { type: string } } - - name: state - in: query - required: false - schema: { type: string } - description: | - RFC 6749 §10.12 marks `state` as RECOMMENDED. Cloud hardening makes it REQUIRED on the initial-entry path (omitted only on the resume path where `oauth_request_id` is supplied instead). This parameter is `required: false` at the spec level only because the operation is dual-mode (initial entry vs. resume); the runtime rejects empty `state` on the initial-entry path with a stable `invalid_request` 400. - - { name: code_challenge, in: query, required: false, schema: { type: string } } - - { name: code_challenge_method, in: query, required: false, schema: { type: string } } - - { name: resource, in: query, required: false, schema: { type: string } } - - { name: oauth_request_id, in: query, required: false, schema: { type: string } } - responses: - "200": - description: Consent challenge payload (session present, email verified). Frontend renders the consent UI from this payload and POSTs back to /oauth/authorize. - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthConsentChallenge" - "302": - description: Redirect to login (no session / unverified email) or to registered redirect_uri (pre-validated client error) - headers: - Location: - schema: - type: string - "400": - description: Invalid authorize request (pre-redirect failure — unknown client, redirect mismatch, malformed params) - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: OAuth disabled - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - post: - operationId: postOAuthAuthorize - tags: [auth] - summary: "[cloud-only] Submit OAuth consent decision" - description: | - [cloud-only] JSON-only consent submission. The handler verifies the per-row CSRF token, atomically marks the authorization request consumed (single-use covers both allow and deny paths), then returns the redirect URL the browser must navigate to. The URL contains either `code` + original `state` for allow, or the RFC 6749 §5.2 error and `state` for deny. - - Workspace membership is re-checked at submission time. Consent is persisted keyed by `(user_id, client_id, resource_id, workspace_id)`; broadening the previously approved scope set requires a fresh consent flow. - x-runtime: [cloud] - security: [] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: [oauth_request_id, csrf_token, decision, workspace_id] - properties: - oauth_request_id: { type: string, format: uuid } - csrf_token: { type: string } - decision: { type: string, enum: [allow, deny] } - workspace_id: { type: string } - responses: - "200": - description: Redirect URL for the frontend to navigate to (allow → with code+state; deny → with error+state) - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthAuthorizeRedirectResponse" - "400": - description: Bad request (CSRF mismatch, expired/consumed request, inaccessible workspace) - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Scope broadening on consent re-grant — fresh consent flow required - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: OAuth disabled - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /oauth/token: - post: - operationId: postOAuthToken - tags: [auth] - summary: "[cloud-only] Exchange authorization code or refresh token for a resource-bound access token" - description: | - [cloud-only] OAuth 2.1 token endpoint (RFC 6749 §3.2). Public clients only — `client_secret` is rejected. - - Two grant types are supported: - - `authorization_code` — exchanges the code minted by `/oauth/authorize` (with PKCE verifier) for an access token + first refresh token. Single-use; reuse fails closed. - - `refresh_token` — rotates the refresh token. Old token immediately invalid; presenting an already-rotated token revokes the entire token family and emits a security metric. - - Both grant types re-validate canonical user state, current workspace membership, and the resource's active flag at every mint. A code or refresh token bound to a deactivated resource fails closed. - - Errors follow RFC 6749 §5.2. Logs never contain raw codes, refresh tokens, or minted tokens. - - Per RFC 6749 §5.1, every 200 and 400 response carries `Cache-Control: no-store` and `Pragma: no-cache` so intermediaries cannot cache token-bearing or state-change-reason responses. - x-runtime: [cloud] - security: [] - requestBody: - required: true - content: - application/x-www-form-urlencoded: - schema: - type: object - required: [grant_type, client_id] - properties: - grant_type: { type: string, enum: [authorization_code, refresh_token] } - client_id: { type: string } - code: { type: string } - redirect_uri: { type: string } - code_verifier: { type: string } - refresh_token: { type: string } - scope: { type: string } - client_secret: { type: string } - responses: - "200": - description: New token pair - headers: - Cache-Control: - schema: - type: string - description: 'Always "no-store" per RFC 6749 §5.1' - Pragma: - schema: - type: string - description: 'Always "no-cache" per RFC 6749 §5.1' - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthTokenResponse" - "400": - description: RFC 6749 §5.2 error - headers: - Cache-Control: - schema: - type: string - description: 'Always "no-store" per RFC 6749 §5.1' - Pragma: - schema: - type: string - description: 'Always "no-cache" per RFC 6749 §5.1' - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthTokenError" - "404": - description: OAuth disabled - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /oauth/register: - post: - operationId: postOAuthRegister - tags: [auth] - summary: "[cloud-only] Dynamic Client Registration (RFC 7591)" - description: | - [cloud-only] Public, unauthenticated, insert-only RFC 7591 §3.1 client registration. Used by MCP-spec-compliant clients to self-register a public OAuth client without operator involvement. - - Policy: - - - Public clients only — `token_endpoint_auth_method` is forced to `none`. Confidential-client registration is out of scope this phase. - - Server-owned `resource_grants`. Caller-supplied `scope` or `resource_grants` is rejected as `invalid_client_metadata` (would be a privilege-escalation surface). Dynamic clients receive the same scopes the active resource publishes. - - Application-type-aware redirect URI policy. `application_type=native` accepts loopback (`127.0.0.1`, `::1`, `localhost`) and reverse-DNS-shaped custom schemes; `application_type=web` accepts HTTPS to hosts in an operator-controlled allowlist only. `application_type` is REQUIRED on the request — missing or empty rejects with `invalid_client_metadata`. - - Anti-impersonation: reserved client names are rejected from third parties via NFKC-folded compare. - - Generated `client_id` carries a stable prefix to distinguish dynamic from seeded clients in audit logs. - - Cache-Control: `no-store` on every 201 and 400 response (the response carries fresh credentials and rejection reasons). - x-runtime: [cloud] - security: [] - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthRegisterRequest" - responses: - "201": - description: Registered. Body echoes the metadata RFC 7591 §3.2.1 requires. - headers: - Cache-Control: - schema: - type: string - description: 'Always "no-store"' - Pragma: - schema: - type: string - description: 'Always "no-cache"' - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthRegisterResponse" - "400": - description: RFC 7591 §3.2.2 invalid client metadata - headers: - Cache-Control: - schema: - type: string - description: 'Always "no-store"' - Pragma: - schema: - type: string - description: 'Always "no-cache"' - content: - application/json: - schema: - $ref: "#/components/schemas/OAuthRegisterError" - "404": - description: OAuth disabled - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "503": - description: No active resource is configured — DCR cannot mint a usable client until an active resource row is seeded. - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - # --------------------------------------------------------------------------- - # Billing (cloud) - # --------------------------------------------------------------------------- - /api/billing/balance: - get: - operationId: getBillingBalance - tags: [billing] - summary: Get current credit balance - description: "[cloud-only] Returns the authenticated user's current credit balance and usage summary." - x-runtime: [cloud] - responses: - "200": - description: Balance info - content: - application/json: - schema: - $ref: "#/components/schemas/BillingBalanceResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/events: - get: - operationId: getBillingEvents - tags: [billing] - summary: List billing events - description: "[cloud-only] Returns a paginated list of billing events (charges, credits, refunds) for the authenticated user." - x-runtime: [cloud] - parameters: - - name: limit - in: query - schema: - type: integer - description: Maximum number of results - - name: offset - in: query - schema: - type: integer - description: Pagination offset - - name: type - in: query - schema: - type: string - description: Filter by event type - responses: - "200": - description: Billing events - content: - application/json: - schema: - $ref: "#/components/schemas/BillingEventsResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/ops/{id}: - get: - operationId: getBillingOpStatus - tags: [billing] - summary: Get a billing operation by ID - description: "[cloud-only] Returns details of a specific billing operation." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: The billing operation ID. - responses: - "200": - description: Billing operation - content: - application/json: - schema: - $ref: "#/components/schemas/BillingOpStatusResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/payment-portal: - post: - operationId: getPaymentPortal - tags: [billing] - summary: Create a payment portal session - description: "[cloud-only] Creates a Stripe customer portal session for managing payment methods and invoices. Returns a URL to redirect the user to." - x-runtime: [cloud] - responses: - "200": - description: Portal session - content: - application/json: - schema: - type: object - properties: - url: - type: string - format: uri - description: Stripe portal URL - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: Bad request (e.g., missing return_url) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/plans: - get: - operationId: getBillingPlans - tags: [billing] - summary: List available billing plans - description: "[cloud-only] Returns the list of available subscription plans and their pricing." - x-runtime: [cloud] - responses: - "200": - description: Plan list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/BillingPlan" - - '401': - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/preview-subscribe: - post: - operationId: previewSubscribe - tags: [billing] - summary: Preview a subscription change - description: "[cloud-only] Returns a preview of what a subscription change would cost, including prorations." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - plan_id - properties: - plan_id: - type: string - description: ID of the plan to preview - responses: - "200": - description: Subscription preview - content: - application/json: - schema: - $ref: "#/components/schemas/PreviewSubscribeResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/status: - get: - operationId: getBillingStatus - tags: [billing] - summary: Get billing status - description: "[cloud-only] Returns the authenticated user's current billing and subscription status." - x-runtime: [cloud] - responses: - "200": - description: Billing status - content: - application/json: - schema: - $ref: "#/components/schemas/BillingStatusResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Workspace not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/subscribe: - post: - operationId: subscribe - tags: [billing] - summary: Subscribe to a billing plan - description: "[cloud-only] Creates a new subscription to the specified billing plan." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - plan_id - properties: - plan_id: - type: string - description: ID of the plan to subscribe to - payment_method_id: - type: string - description: Stripe payment method ID - responses: - "200": - description: Subscription created - content: - application/json: - schema: - $ref: "#/components/schemas/SubscribeResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/subscription/cancel: - post: - operationId: cancelSubscription - tags: [billing] - summary: Cancel the active subscription - description: "[cloud-only] Cancels the authenticated user's active subscription. The subscription remains active until the end of the current billing period." - x-runtime: [cloud] - responses: - "200": - description: Subscription cancelled - content: - application/json: - schema: - $ref: "#/components/schemas/CancelSubscriptionResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: Invalid request (e.g., no active subscription) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/subscription/resubscribe: - post: - operationId: resubscribe - tags: [billing] - summary: Resubscribe after cancellation - description: "[cloud-only] Reactivates a subscription that was previously cancelled but has not yet expired." - x-runtime: [cloud] - responses: - "200": - description: Subscription reactivated - content: - application/json: - schema: - $ref: "#/components/schemas/ResubscribeResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: Invalid request (e.g., no active subscription, not in cancellation grace period) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/billing/topup: - post: - operationId: createTopup - tags: [billing] - summary: Purchase additional credits - description: "[cloud-only] Purchases a one-time credit top-up using the user's payment method on file." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - amount - properties: - amount: - type: integer - description: Number of credits to purchase - responses: - "200": - description: Top-up successful - content: - application/json: - schema: - $ref: "#/components/schemas/CreateTopupResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # Workspace (cloud) - # --------------------------------------------------------------------------- - /api/workspace/api-keys: - get: - operationId: listWorkspaceAPIKeys - tags: [workspace] - summary: List workspace API keys - description: "[cloud-only] Returns the list of API keys for the current workspace." - x-runtime: [cloud] - responses: - "200": - description: API key list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/WorkspaceApiKey" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: createWorkspaceAPIKey - tags: [workspace] - summary: Create a workspace API key - description: "[cloud-only] Creates a new API key for the current workspace." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - name - properties: - name: - type: string - description: Display name for the API key - description: - type: string - description: User-provided description of the key's purpose - maxLength: 5000 - responses: - "201": - description: API key created - content: - application/json: - schema: - $ref: "#/components/schemas/CreateWorkspaceAPIKeyResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Workspace not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '429': - description: Key limit reached - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/api-keys/{id}: - delete: - operationId: revokeWorkspaceAPIKey - tags: [workspace] - summary: Delete a workspace API key - description: "[cloud-only] Revokes and deletes a workspace API key." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: The API key ID. - responses: - "204": - description: API key deleted - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/invites: - get: - operationId: listWorkspaceInvites - tags: [workspace] - summary: List pending workspace invites - description: "[cloud-only] Returns the list of pending invitations for the current workspace." - x-runtime: [cloud] - responses: - "200": - description: Invite list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/WorkspaceInvite" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: createWorkspaceInvite - tags: [workspace] - summary: Invite a user to the workspace - description: "[cloud-only] Creates an invitation for a user to join the current workspace." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - email - properties: - email: - type: string - format: email - description: Email address to invite - role: - type: string - enum: [admin, member] - description: Role to assign - responses: - "201": - description: Invite created - content: - application/json: - schema: - $ref: "#/components/schemas/PendingInvite" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "409": - description: Conflict - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Workspace not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/invites/{inviteId}: - delete: - operationId: revokeWorkspaceInvite - tags: [workspace] - summary: Cancel a workspace invite - description: "[cloud-only] Cancels a pending workspace invitation." - x-runtime: [cloud] - parameters: - - name: inviteId - in: path - required: true - schema: - type: string - description: The invite ID. - responses: - "204": - description: Invite cancelled - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/leave: - post: - operationId: leaveWorkspace - tags: [workspace] - summary: Leave the current workspace - description: "[cloud-only] Removes the authenticated user from the current workspace." - x-runtime: [cloud] - responses: - "204": - description: Left workspace - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Workspace not found or not a member - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/members: - get: - operationId: listWorkspaceMembers - tags: [workspace] - summary: List workspace members - description: "[cloud-only] Returns the list of members in the current workspace." - x-runtime: [cloud] - responses: - "200": - description: Member list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/WorkspaceMember" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Workspace not found - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/members/{user_id}/api-keys: - get: - operationId: listMemberApiKeys - tags: [workspace] - summary: List API keys for a workspace member - description: "[cloud-only] Returns the API keys belonging to a specific workspace member. Requires admin role." - x-runtime: [cloud] - parameters: - - name: user_id - in: path - required: true - schema: - type: string - description: The member's user ID. - responses: - "200": - description: API key list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/WorkspaceApiKey" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - delete: - operationId: bulkRevokeWorkspaceMemberAPIKeys - tags: [workspace] - summary: Bulk revoke a member's API keys - description: "[cloud-only] Revokes all active API keys for a specific workspace member. Only workspace owners can perform this action." - x-runtime: [cloud] - parameters: - - name: user_id - in: path - required: true - schema: - type: string - minLength: 1 - description: The member's user ID. - responses: - "200": - description: Keys revoked - content: - application/json: - schema: - $ref: "#/components/schemas/BulkRevokeAPIKeysResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden — must be workspace owner - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '422': - description: Validation error (e.g. empty user_id) - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspace/members/{userId}: - patch: - operationId: updateWorkspaceMember - tags: [workspace] - summary: Update a workspace member's role - description: "[cloud-only] Updates the role of a workspace member. Requires admin role." - x-runtime: [cloud] - parameters: - - name: userId - in: path - required: true - schema: - type: string - description: The member's user ID. - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - role - properties: - role: - type: string - enum: [admin, member] - description: New role to assign - responses: - "200": - description: Member updated - content: - application/json: - schema: - $ref: "#/components/schemas/WorkspaceMember" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - delete: - operationId: removeWorkspaceMember - tags: [workspace] - summary: Remove a member from the workspace - description: "[cloud-only] Removes a member from the current workspace. Requires admin role." - x-runtime: [cloud] - parameters: - - name: userId - in: path - required: true - schema: - type: string - description: The member's user ID. - responses: - "204": - description: Member removed - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspaces: - get: - operationId: listWorkspaces - tags: [workspace] - summary: List workspaces the user belongs to - description: "[cloud-only] Returns the list of workspaces the authenticated user is a member of." - x-runtime: [cloud] - responses: - "200": - description: Workspace list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Workspace" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '404': - description: Feature not enabled for user - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: createWorkspace - tags: [workspace] - summary: Create a new workspace - description: "[cloud-only] Creates a new workspace. The authenticated user becomes the owner." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - name - properties: - name: - type: string - description: Workspace name - responses: - "201": - description: Workspace created - content: - application/json: - schema: - $ref: "#/components/schemas/Workspace" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '404': - description: Feature not enabled for user - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/workspaces/{id}: - get: - operationId: getWorkspace - tags: [workspace] - summary: Get a workspace by ID - description: "[cloud-only] Returns details of a workspace the user is a member of." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: The workspace ID. - responses: - "200": - description: Workspace detail - content: - application/json: - schema: - $ref: "#/components/schemas/Workspace" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - patch: - operationId: updateWorkspace - tags: [workspace] - summary: Update workspace settings - description: "[cloud-only] Updates the name or settings of a workspace. Requires admin role." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: The workspace ID. - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - name: - type: string - description: New workspace name - responses: - "200": - description: Workspace updated - content: - application/json: - schema: - $ref: "#/components/schemas/Workspace" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: deleteWorkspace - tags: [workspace] - summary: Delete a workspace - description: "[cloud-only] Soft-deletes a workspace. Requires owner role. Personal workspaces cannot be deleted." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: The workspace ID. - responses: - "204": - description: Workspace deleted - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "403": - description: Forbidden — must be workspace owner - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - # --------------------------------------------------------------------------- - # User / settings / misc (cloud) - # --------------------------------------------------------------------------- - /api/feedback: - post: - operationId: submitFeedback - tags: [user] - summary: Submit user feedback - description: "[cloud-only] Submits feedback from the user about their experience with the cloud runtime." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/FeedbackRequest" - responses: - "201": - description: Feedback submitted - content: - application/json: - schema: - type: object - properties: - id: - type: string - status: - type: string - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/files/mask-layers: - get: - operationId: getMaskLayers - tags: [assets] - summary: Get related mask layer filenames - description: "[cloud-only] Given a mask file (any of the 4 layers), returns all related mask layer filenames. Used by the mask editor to load the paint, mask, and painted layers when reopening a previously edited mask." - x-runtime: [cloud] - parameters: - - name: filename - in: query - required: true - schema: - type: string - description: Hash filename of any mask layer file - responses: - "200": - description: Related mask layers - content: - application/json: - schema: - type: object - properties: - mask: - type: string - description: Filename of the mask layer - nullable: true - paint: - type: string - description: Filename of the paint strokes layer - nullable: true - painted: - type: string - description: Filename of the painted image layer - nullable: true - painted_masked: - type: string - description: Filename of the final composite layer - nullable: true - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: File not found or not a mask file - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/internal/cloud_analytics: - post: - operationId: postCloudAnalytics - tags: [internal] - summary: Post client analytics events - description: "[cloud-only] Receives analytics events from the frontend for processing by the cloud analytics pipeline." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - events - properties: - events: - type: array - items: - type: object - required: - - event_name - properties: - event_name: - type: string - timestamp: - type: string - format: date-time - properties: - type: object - additionalProperties: true - responses: - "200": - description: Events accepted - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/invites/{token}/accept: - post: - operationId: acceptWorkspaceInvite - tags: [workspace] - summary: Accept a workspace invitation - description: "[cloud-only] Accepts a workspace invitation using the invite token. The authenticated user is added to the workspace." - x-runtime: [cloud] - parameters: - - name: token - in: path - required: true - schema: - type: string - description: The invitation token. - responses: - "200": - description: Invite accepted - content: - application/json: - schema: - $ref: "#/components/schemas/AcceptInviteResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '403': - description: Email does not match invite - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '409': - description: Already a member of this workspace - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/secrets: - get: - operationId: listSecrets - tags: [settings] - summary: List user secrets - description: "[cloud-only] Returns the list of secrets (API keys for third-party services) stored for the authenticated user. Secret values are redacted." - x-runtime: [cloud] - responses: - "200": - description: Secret list - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/SecretMeta" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '503': - description: Service unavailable - feature is disabled - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: createSecret - tags: [settings] - summary: Create or update a secret - description: "[cloud-only] Stores a new secret or updates an existing one. Secrets are encrypted at rest." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - name - - value - properties: - name: - type: string - description: Secret name (unique per user) - value: - type: string - description: Secret value - responses: - "201": - description: Secret created - content: - application/json: - schema: - $ref: "#/components/schemas/SecretResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '409': - description: Conflict - secret with this name or provider already exists - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '422': - description: Validation error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '503': - description: Service unavailable - secrets feature disabled - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/secrets/{id}: - get: - operationId: getSecret - tags: [settings] - summary: Get secret metadata - description: "[cloud-only] Returns metadata for a specific secret. Does not return the plaintext secret value." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - format: uuid - description: The secret ID. - responses: - "200": - description: Secret metadata - content: - application/json: - schema: - $ref: "#/components/schemas/SecretResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '403': - description: Forbidden - user does not own this secret - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '503': - description: Service unavailable - secrets feature disabled - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - patch: - operationId: updateSecret - tags: [settings] - summary: Update a secret - description: "[cloud-only] Updates an existing secret's name and/or value. Both fields are optional; only provided fields are updated." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - format: uuid - description: The secret ID. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/UpdateSecretRequest" - responses: - "200": - description: Secret updated - content: - application/json: - schema: - $ref: "#/components/schemas/SecretResponse" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "409": - description: Conflict — a secret with this name already exists - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '403': - description: Forbidden - user does not own this secret - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '503': - description: Service unavailable - secrets feature disabled - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - delete: - operationId: deleteSecret - tags: [settings] - summary: Delete a secret - description: "[cloud-only] Permanently deletes a stored secret." - x-runtime: [cloud] - parameters: - - name: id - in: path - required: true - schema: - type: string - description: The secret ID. - responses: - "204": - description: Secret deleted - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '403': - description: Forbidden - user does not own this secret - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '503': - description: Service unavailable - secrets feature disabled - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/user: - get: - operationId: getUser - tags: [user] - summary: Get the authenticated cloud user - description: "[cloud-only] Returns the profile and account information for the currently authenticated user." - x-runtime: [cloud] - responses: - "200": - description: User profile - content: - application/json: - schema: - $ref: "#/components/schemas/UserResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - put: - operationId: updateCloudUser - tags: [user] - summary: Update the authenticated cloud user profile - description: "[cloud-only] Updates the profile information for the currently authenticated user." - x-runtime: [cloud] - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - display_name: - type: string - avatar_url: - type: string - format: uri - responses: - "200": - description: Updated profile - content: - application/json: - schema: - $ref: "#/components/schemas/CloudUser" - "400": - description: Bad request - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/userdata/{file}/publish: - get: - operationId: getUserdataFilePublish - tags: [userdata] - summary: Get publish info for a userdata file - description: "[cloud-only] Returns the publish status and share info for a userdata workflow file." - x-runtime: [cloud] - parameters: - - name: file - in: path - required: true - schema: - type: string - description: File path relative to user data directory - responses: - "200": - description: Publish info (publish_time is null if never published) - content: - application/json: - schema: - $ref: "#/components/schemas/WorkflowPublishInfo" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Workflow not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - post: - operationId: postUserdataFilePublish - tags: [userdata] - summary: Publish a userdata file to the cloud - description: "[cloud-only] Makes a userdata file available via a public URL for sharing or embedding." - x-runtime: [cloud] - parameters: - - name: file - in: path - required: true - schema: - type: string - description: File path relative to user data directory - responses: - "200": - description: Published file URL - content: - application/json: - schema: - type: object - properties: - url: - type: string - format: uri - description: Public URL of the published file - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: Bad request - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/vhs/queryvideo: - get: - operationId: getVhsQueryVideo - tags: [view] - summary: Query VHS video metadata - description: "[cloud-only] Returns metadata about a video file processed by the VHS (Video Helper Suite) integration." - x-runtime: [cloud] - parameters: - - name: filename - in: query - required: true - schema: - type: string - description: Video filename - - name: type - in: query - schema: - type: string - enum: [input, output, temp] - description: Directory type - - name: subfolder - in: query - schema: - type: string - description: Subfolder within the directory - responses: - "200": - description: Video metadata - content: - application/json: - schema: - type: object - additionalProperties: true - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '400': - description: 'Missing required query parameter. Produced by the oapi-codegen - wrapper via echo.NewHTTPError, so the body shape matches Echo''s - default HTTPError serialization rather than ErrorResponse. - ' - content: - application/json: - schema: - $ref: '#/components/schemas/BindingErrorResponse' - /api/vhs/viewaudio: - get: - operationId: viewVhsAudio - tags: [view] - summary: View or download VHS audio - description: "[cloud-only] Returns audio content from a VHS-processed file." - x-runtime: [cloud] - parameters: - - name: filename - in: query - required: true - schema: - type: string - description: Audio filename - - name: type - in: query - schema: - type: string - enum: [input, output, temp] - description: Directory type - - name: subfolder - in: query - schema: - type: string - description: Subfolder within the directory - responses: - "200": - description: Audio content - content: - audio/*: - schema: - type: string - format: binary - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/vhs/viewvideo: - get: - operationId: viewVhsVideo - tags: [view] - summary: View or download VHS video - description: "[cloud-only] Returns video content from a VHS-processed file." - x-runtime: [cloud] - parameters: - - name: filename - in: query - required: true - schema: - type: string - description: Video filename - - name: type - in: query - schema: - type: string - enum: [input, output, temp] - description: Directory type - - name: subfolder - in: query - schema: - type: string - description: Subfolder within the directory - responses: - "200": - description: Video content - content: - video/*: - schema: - type: string - format: binary - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/viewvideo: - get: - operationId: viewVideo - tags: [view] - summary: View or download a video file - deprecated: true - description: | - **Deprecated.** This endpoint is an alias of `GET /api/view` added for - legacy history-queue video playback. Callers should use `/api/view` - directly; the endpoint is retained for backward compatibility but will - be removed in a future release. - x-runtime: [cloud] - parameters: - - name: filename - in: query - required: true - schema: - type: string - description: Video filename - - name: type - in: query - schema: - type: string - enum: [input, output, temp] - description: Directory type - - name: subfolder - in: query - schema: - type: string - description: Subfolder within the directory - responses: - "200": - description: Video content - content: - video/*: - schema: - type: string - format: binary - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - /api/tasks: - get: - operationId: listTasks - tags: [task] - summary: List background tasks - description: "[cloud-only] Retrieve a paginated list of background tasks for the authenticated user. Supports filtering by task type, status, and creation time." - x-runtime: [cloud] - parameters: - - name: task_name - in: query - schema: - type: string - description: Filter by task type name (exact match). - - name: idempotency_key - in: query - schema: - type: string - description: Filter by idempotency key (exact match). - - name: status - in: query - schema: - type: string - description: Filter by one or more statuses (comma-separated). - - name: created_after - in: query - schema: - type: string - format: date-time - description: Filter tasks created after this timestamp. - - name: created_before - in: query - schema: - type: string - format: date-time - description: Filter tasks created before this timestamp. - - name: sort_order - in: query - schema: - type: string - enum: [asc, desc] - default: desc - description: Sort direction by create_time. - - name: offset - in: query - schema: - type: integer - minimum: 0 - default: 0 - description: Pagination offset (0-based). - - name: limit - in: query - schema: - type: integer - minimum: 1 - maximum: 100 - default: 20 - description: Maximum items per page (1-100). - responses: - "200": - description: Tasks retrieved - content: - application/json: - schema: - $ref: "#/components/schemas/TasksListResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "422": - description: Validation error - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - /api/tasks/{task_id}: - get: - operationId: getTask - tags: [task] - summary: Get task details - description: "[cloud-only] Retrieve full details for a specific background task." - x-runtime: [cloud] - parameters: - - name: task_id - in: path - required: true - schema: - type: string - format: uuid - description: Task identifier (UUID). - responses: - "200": - description: Task details - content: - application/json: - schema: - $ref: "#/components/schemas/TaskResponse" - "401": - description: Unauthorized - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - "404": - description: Task not found - content: - application/json: - schema: - $ref: "#/components/schemas/CloudError" - - '500': - description: Internal server error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' components: - parameters: - ComfyUserHeader: - name: Comfy-User - in: header - required: false - schema: - type: string - description: | - Identifies the active user in multi-user mode. Used for settings, - userdata, and history isolation. This is not a security mechanism — - it is an organisational convenience with no authentication behind it. - - schemas: - # ------------------------------------------------------------------- - # Prompt - # ------------------------------------------------------------------- - PromptRequest: - type: object - description: A workflow submission. Wraps the prompt graph plus optional client identifier and extra per-request data. - required: - - prompt - properties: - prompt: - type: object - description: | - The workflow graph to execute. Keys are node IDs (strings); - values are objects with class_type and inputs. - additionalProperties: true - number: - type: number - description: Priority number for the queue (lower numbers have higher priority) - front: - type: boolean - description: If true, adds the prompt to the front of the queue - extra_data: - type: object - description: Extra data associated with the prompt (e.g. extra_pnginfo) - additionalProperties: true - client_id: - type: string - description: WebSocket client ID to receive progress updates - prompt_id: - type: string - format: uuid - description: "Client-supplied prompt ID. Server generates a UUID if omitted." - partial_execution_targets: - type: array - items: - type: string - description: List of node IDs to execute (partial graph execution) - workflow_id: - type: string - format: uuid - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Cloud workflow entity ID for tracking and gallery association. Ignored by local ComfyUI." - workflow_version_id: - type: string - format: uuid - nullable: true - x-runtime: [cloud] - description: "[cloud-only] Cloud workflow version ID for pinning execution to a specific version. Ignored by local ComfyUI." - - PromptResponse: - type: object - description: Server acknowledgement of a workflow submission. Includes the assigned `prompt_id` and current queue position. - properties: - prompt_id: - type: string - format: uuid - description: Unique identifier for the prompt execution - number: - type: number - description: Priority number in the queue - node_errors: - type: object - description: Validation errors keyed by node ID - additionalProperties: - $ref: "#/components/schemas/NodeError" - error: - description: Top-level prompt error (string message or structured error) - oneOf: - - type: string - - $ref: "#/components/schemas/PromptError" - - PromptErrorResponse: - type: object - description: Error response when prompt validation fails - additionalProperties: true - - PromptError: - type: object - description: Structured prompt validation error - properties: - type: - type: string - message: - type: string - details: - type: string - - Error: - type: object - description: Detailed node-level error - properties: - type: - type: string - message: - type: string - details: - type: string - extra_info: - type: object - properties: - input_name: - type: string - additionalProperties: true - - NodeError: - type: object - description: Error details for a single node - properties: - errors: - type: array - items: - $ref: "#/components/schemas/Error" - class_type: - type: string - description: The node's class type - dependent_outputs: - type: array - items: {} - - PromptInfo: - type: object - description: Summary of a queued or recently-executed prompt, as returned by the queue and history endpoints. - properties: - exec_info: - type: object - properties: - queue_remaining: - type: integer - description: Number of items remaining in the queue - - # ------------------------------------------------------------------- - # Queue - # ------------------------------------------------------------------- - QueueInfo: - type: object - description: Queue information with pending and running items - properties: - queue_running: - type: array - description: Currently running queue items - items: - type: array - description: | - Queue item tuple: [number, prompt_id, prompt, extra_data, outputs_to_execute, sensitive] - items: {} - prefixItems: - - type: number - description: Priority number - - type: string - format: uuid - description: prompt_id - - type: object - description: prompt graph - additionalProperties: true - - type: object - description: extra_data - additionalProperties: true - - type: array - description: outputs_to_execute (list of output node IDs) - items: - type: string - - type: object - description: sensitive data (may be omitted) - additionalProperties: true - queue_pending: - type: array - description: Pending queue items (oldest first) - items: - type: array - description: | - Queue item tuple: [number, prompt_id, prompt, extra_data, outputs_to_execute, sensitive] - items: {} - prefixItems: - - type: number - description: Priority number - - type: string - format: uuid - description: prompt_id - - type: object - description: prompt graph - additionalProperties: true - - type: object - description: extra_data - additionalProperties: true - - type: array - description: outputs_to_execute (list of output node IDs) - items: - type: string - - type: object - description: sensitive data (may be omitted) - additionalProperties: true - - QueueManageRequest: - type: object - description: Request to clear or delete from queue - properties: - clear: - type: boolean - description: If true, clear all pending items - delete: - type: array - items: - type: string - description: Array of prompt IDs to delete from queue - - QueueManageResponse: - type: object - x-runtime: [cloud] - description: >- - [cloud-only] Result of a queue mutation. The Cloud runtime returns which - items were deleted and whether the queue was cleared; local ComfyUI - returns an empty 200 body. - properties: - deleted: - type: array - nullable: true - items: - type: string - description: Prompt IDs that were deleted from the queue. - cleared: - type: boolean - nullable: true - description: Whether the queue was cleared. - - # ------------------------------------------------------------------- - # History - # ------------------------------------------------------------------- - HistoryEntry: - type: object - description: A single execution history entry - properties: - prompt: - type: array - description: | - Prompt tuple: [number, prompt_id, prompt_graph, extra_data, output_node_ids] - items: {} - outputs: - type: object - description: Output data from execution keyed by node ID - additionalProperties: true - status: - type: object - description: Execution status (status_str, completed, messages, etc.) - additionalProperties: true - meta: - type: object - description: Metadata about the execution and nodes - additionalProperties: true - - HistoryManageRequest: - type: object - description: Request to clear or delete history entries - properties: - clear: - type: boolean - description: If true, clear all history - delete: - type: array - items: - type: string - description: Array of prompt IDs to delete from history - - # ------------------------------------------------------------------- - # Jobs - # ------------------------------------------------------------------- - JobEntry: - type: object - description: Lightweight job data for list views - required: - - id - - status - properties: - id: - type: string - format: uuid - description: Unique job identifier (same as prompt_id) - status: - type: string - enum: - - pending - - in_progress - - completed - - failed - - cancelled - description: Current job status - create_time: - type: integer - format: int64 - description: Job creation timestamp (Unix milliseconds). - execution_start_time: - type: integer - format: int64 - description: Workflow execution start timestamp (Unix milliseconds, terminal states only). - execution_end_time: - type: integer - format: int64 - description: Workflow execution end timestamp (Unix milliseconds, terminal states only). - preview_output: - type: object - additionalProperties: true - description: Primary preview output - outputs_count: - type: integer - description: Total number of output files - workflow_id: - type: string - nullable: true - x-runtime: [cloud] - description: "[cloud-only] UUID of the Cloud workflow entity this job is associated with. Local ComfyUI returns null." - execution_error: - x-runtime: [cloud] - description: "[cloud-only] Detailed execution error from ComfyUI for failed jobs. Absent on local ComfyUI." - allOf: - - $ref: "#/components/schemas/ExecutionError" - - JobDetailResponse: - type: object - description: Full job details including workflow and outputs - required: - - id - - status - properties: - id: - type: string - format: uuid - status: - type: string - enum: - - pending - - in_progress - - completed - - failed - - cancelled - workflow: - type: object - additionalProperties: true - description: Full ComfyUI workflow - outputs: - type: object - additionalProperties: true - description: Full outputs object from execution - execution_error: - $ref: "#/components/schemas/ExecutionError" - create_time: - type: integer - format: int64 - description: Job creation timestamp (Unix milliseconds). - update_time: - type: integer - format: int64 - description: Last state-change timestamp (Unix milliseconds). - execution_start_time: - type: integer - format: int64 - description: Workflow execution start timestamp (Unix milliseconds, terminal states only). - execution_end_time: - type: integer - format: int64 - description: Workflow execution end timestamp (Unix milliseconds, terminal states only). - preview_output: - type: object - additionalProperties: true - outputs_count: - type: integer - execution_status: - type: object - additionalProperties: true - execution_meta: - type: object - additionalProperties: true - - ExecutionError: - type: object - description: Detailed execution error from ComfyUI - properties: - node_id: - type: string - description: ID of the node that failed - node_type: - type: string - description: Type name of the node - exception_message: - type: string - description: Human-readable error message - exception_type: - type: string - description: Python exception type - traceback: - type: array - items: - type: string - description: Traceback lines - current_inputs: - type: object - additionalProperties: true - current_outputs: - type: object - additionalProperties: true - - PaginationInfo: - type: object - description: Pagination metadata returned alongside list responses. - properties: - offset: - type: integer - limit: - type: integer - total: - type: integer - has_more: - type: boolean - - # ------------------------------------------------------------------- - # Upload / View - # ------------------------------------------------------------------- - UploadResult: - type: object - description: Response body returned by the image/mask upload endpoints, describing where the uploaded file now lives. - properties: - name: - type: string - description: Saved filename (may be renamed to avoid collisions) - subfolder: - type: string - description: Subfolder the file was saved to - type: - type: string - description: Directory type (input, temp) - - # ------------------------------------------------------------------- - # System - # ------------------------------------------------------------------- - DeviceStats: - type: object - description: GPU/compute device statistics - required: - - name - - type - - index - properties: - name: - type: string - description: Device name - type: - type: string - description: Device type (cuda, mps, cpu, etc.) - index: - type: number - nullable: true - description: | - Device index within its type (e.g. CUDA ordinal for `cuda:0`, - `cuda:1`). `null` for devices with no index, including the CPU - device returned in `--cpu` mode (PyTorch's `torch.device('cpu').index` - is `None`). - vram_total: - type: number - description: Total VRAM in bytes - vram_free: - type: number - description: Free VRAM in bytes - torch_vram_total: - type: number - description: Total PyTorch-managed VRAM in bytes - torch_vram_free: - type: number - description: Free PyTorch-managed VRAM in bytes - - SystemStatsResponse: - type: object - description: Hardware, VRAM, Python, and ComfyUI version information for the running process. - required: - - system - - devices - properties: - system: - type: object - required: - - os - - python_version - - embedded_python - - comfyui_version - - pytorch_version - - argv - - ram_total - - ram_free - properties: - os: - type: string - description: Operating system - python_version: - type: string - description: Python version - embedded_python: - type: boolean - description: Whether using embedded Python - comfyui_version: - type: string - description: ComfyUI version string - pytorch_version: - type: string - description: PyTorch version - required_frontend_version: - type: string - description: Required frontend version - argv: - type: array - items: - type: string - description: Command line arguments - ram_total: - type: number - description: Total RAM in bytes - ram_free: - type: number - description: Free RAM in bytes - installed_templates_version: - type: string - nullable: true - description: Version of the currently installed workflow templates - required_templates_version: - type: string - nullable: true - description: Minimum required workflow templates version for this ComfyUI build - comfy_package_versions: - type: array - description: Installed and required versions for every comfy* package pinned in requirements.txt - items: - type: object - required: - - name - - installed - - required - properties: - name: - type: string - installed: + schemas: + Asset: + description: Represents a user-owned asset (image, video, or other generated output). + properties: + created_at: + description: Timestamp when the asset was created + format: date-time type: string + display_name: + description: Display name of the asset. Mirrors name for backwards compatibility. nullable: true + type: string + file_path: + description: Relative path in global-namespace-root form (e.g. "models/checkpoints/flux.safetensors") + nullable: true + type: string + hash: + description: Blake3 hash of the asset content. + pattern: ^blake3:[a-f0-9]{64}$ + type: string + id: + description: Unique identifier for the asset + format: uuid + type: string + is_immutable: + description: Whether this asset is immutable (cannot be modified or deleted) + type: boolean + job_id: + description: ID of the job that created this asset, if available + format: uuid + nullable: true + type: string + last_access_time: + description: Timestamp when the asset was last accessed + format: date-time + type: string + metadata: + additionalProperties: true + description: System-managed metadata from download sources (HuggingFace, CivitAI, etc.) - read-only, not user-modifiable + readOnly: true + type: object + mime_type: + description: MIME type of the asset + type: string + name: + description: Name of the asset file + type: string + preview_id: + description: ID of the preview asset if available + format: uuid + nullable: true + type: string + preview_url: + description: URL for asset preview/thumbnail + format: uri + type: string + short_url: + description: Durable, owner-gated short link to this asset's content (relative `/api/s/{id}` path). Stable across the underlying signed URL's expiry — resolving it re-mints a fresh signed URL on every request — so it is safe to persist or share into chat, unlike `preview_url`. Only the minting user can resolve it. Omitted when the short-link surface is disabled or the asset has no resolvable content hash. + nullable: true + type: string + x-runtime: + - cloud + size: + description: Size of the asset in bytes + format: int64 + type: integer + tags: + description: Tags associated with the asset + items: + type: string + type: array + updated_at: + description: Timestamp when the asset was last updated + format: date-time + type: string + user_metadata: + additionalProperties: true + description: Custom user metadata for the asset + type: object + required: + - id + - name + - created_at + - updated_at + type: object + AssetCreated: + allOf: + - $ref: '#/components/schemas/Asset' + - properties: + created_new: + description: Whether this was a new asset creation (true) or returned existing (false) + type: boolean required: + - created_new + type: object + description: Response returned when a new asset is successfully created. + AssetInfo: + description: Lightweight asset reference used in workflow publishing payloads. + properties: + id: + description: Asset identifier. type: string + in_library: + description: Whether the caller already owns this asset. + type: boolean + model: + description: Whether this asset is a model. + type: boolean + name: + type: string + preview_url: + description: Signed URL for previewing the asset. + type: string + public: + description: Whether this is a public (platform-provided) asset. + type: boolean + storage_url: + type: string + required: + - id + - name + - preview_url + - storage_url + - model + - public + - in_library + type: object + AssetTagHistogramResponse: + description: Histogram of tag counts used for refining asset search results. + properties: + tag_counts: + additionalProperties: + type: integer + description: Map of tag names to their occurrence counts on matching assets + example: + checkpoint: 32 + lora: 193 + vae: 6 + type: object + required: + - tag_counts + type: object + AssetUpdated: + description: Response returned when an existing asset is successfully updated. + properties: + display_name: + description: Display name of the asset. Mirrors name for backwards compatibility. nullable: true - devices: - type: array - items: - $ref: "#/components/schemas/DeviceStats" - - # ------------------------------------------------------------------- - # Node / Object Info - # ------------------------------------------------------------------- - NodeInfo: - type: object - description: 'Definition of a registered node class: its inputs, outputs, category, and display metadata.' - properties: - input: - type: object - description: Input specifications (required and optional groups) - additionalProperties: true - input_order: - type: object - description: Ordered input names per group - additionalProperties: - type: array - items: - type: string - output: - type: array - items: - type: string - description: Output type names - output_is_list: - type: array - items: - type: boolean - description: Whether each output is a list - output_name: - type: array - items: - type: string - description: Display names of outputs - name: - type: string - description: Internal class name - display_name: - type: string - description: Human-readable display name - description: - type: string - description: Node description - python_module: - type: string - description: Python module implementing the node - category: - type: string - description: Node category path - output_node: - type: boolean - description: Whether this is an output node - output_tooltips: - type: array - items: - type: string - description: Tooltips for each output - deprecated: - type: boolean - description: Whether the node is deprecated - experimental: - type: boolean - description: Whether the node is experimental - api_node: - type: boolean - description: Whether this is an API node - is_input_list: - type: boolean - description: Whether the node accepts list inputs - dev_only: - type: boolean - description: Whether the node is developer-only (hidden in production UI) - has_intermediate_output: - type: boolean - description: Whether the node emits intermediate output during execution - search_aliases: - type: array - items: - type: string - description: Alternative search terms for finding this node - essentials_category: - type: string - nullable: true - description: | - Category override used by the essentials pack. The - `essentials_category` key may be present with a string value, - present and `null`, or absent entirely: - - - V1 nodes: `essentials_category` is **omitted** when the node - class doesn't define an `ESSENTIALS_CATEGORY` attribute, and - **`null`** if the attribute is explicitly set to `None`. - - V3 nodes (`comfy_api.latest.io`): `essentials_category` is - **always present**, and **`null`** for nodes whose `Schema` - doesn't populate it. - - # ------------------------------------------------------------------- - # Models - # ------------------------------------------------------------------- - ModelFolder: - type: object - description: A configured model folder and the list of disk paths it resolves to. - required: - - name - - folders - properties: - name: - type: string - description: Model folder type name (e.g. "checkpoints") - folders: - type: array - items: - type: string - description: Filesystem paths for this model type - - ModelFile: - type: object - description: A single model file in a folder, with filesystem metadata. - required: - - name - - pathIndex - properties: - name: - type: string - description: Model filename - pathIndex: - type: integer - description: Index into the folder's paths array - modified: - type: number - description: File modification timestamp - created: - type: number - description: File creation timestamp - size: - type: integer - format: int64 - description: File size in bytes - - # ------------------------------------------------------------------- - # Subgraphs - # ------------------------------------------------------------------- - GlobalSubgraphInfo: - type: object - description: Metadata for a global subgraph blueprint (without full data) - required: - - source - - name - - info - properties: - source: - type: string - description: Source type ("templates" or "custom_node") - name: - type: string - description: Display name of the subgraph blueprint - info: - type: object - description: Additional information about the subgraph - required: - - node_pack - properties: - node_pack: - type: string - description: The node pack/module providing this subgraph - data: - type: string - description: The full subgraph JSON data (may be empty in list view) - - GlobalSubgraphData: - type: object - description: Full data for a global subgraph blueprint - required: - - source - - name - - info - - data - properties: - source: - type: string - description: Source type ("templates" or "custom_node") - name: - type: string - description: Display name of the subgraph blueprint - info: - type: object - description: Additional information about the subgraph - required: - - node_pack - properties: - node_pack: - type: string - description: The node pack/module providing this subgraph - data: - type: string - description: The full subgraph JSON data as a string - - # ------------------------------------------------------------------- - # Userdata - # ------------------------------------------------------------------- - UserDataResponse: - description: | - Response body for the POST endpoints `/api/userdata/{file}` and - `/api/userdata/{file}/move/{dest}`. Returns a single item whose - shape depends on the `full_info` query parameter. - x-variant-selector: - full_info=true: file-info object (`GetUserDataResponseFullFile`) - default: relative path string - oneOf: - - $ref: "#/components/schemas/GetUserDataResponseFullFile" - - type: string - description: Relative path of the written or moved file. Returned when `full_info` is absent or false. - - ListUserdataResponse: - description: | - Response body for `GET /api/userdata`. The array item shape is - determined by the `full_info` and `split` query parameters. - x-variant-selector: - full_info=true: array of file-info objects (`GetUserDataResponseFullFile`) - split=true: array of `[relative_path, ...path_components]` arrays - default: array of relative path strings - oneOf: - - type: array - items: - $ref: "#/components/schemas/GetUserDataResponseFullFile" - description: Returned when `full_info=true`. - - type: array - items: - type: array - items: - type: string - minItems: 2 - description: | - Returned when `split=true` and `full_info=false`. Each inner - array is `[relative_path, ...path_components]`. - - type: array - items: - type: string - description: Default shape — array of file paths relative to the user data root. - - GetUserDataResponseFullFile: - type: object - description: A single entry in a full-info user data listing. - properties: - path: - type: string - description: File name or path relative to the user directory - created: - type: number - description: Unix timestamp of file creation - size: - type: integer - description: File size in bytes - modified: - type: integer - format: int64 - description: Unix timestamp of last modification in milliseconds - - # ------------------------------------------------------------------- - # Assets - # ------------------------------------------------------------------- - Asset: - type: object - description: A registered asset — an input/output file tracked in the asset database with content hash and metadata. - required: - - id - - name - - size - - created_at - - updated_at - properties: - id: - type: string - format: uuid - description: Unique identifier for the asset - name: - type: string - description: Name of the asset file - hash: - type: string - nullable: true - description: Blake3 content hash of the asset (preferred over asset_hash) - pattern: "^blake3:[a-f0-9]{64}$" - asset_hash: - type: string - nullable: true - deprecated: true - description: "Deprecated: use `hash` instead. Blake3 hash of the asset content." - pattern: "^blake3:[a-f0-9]{64}$" - size: - type: integer - format: int64 - description: Size of the asset in bytes - mime_type: - type: string - description: MIME type of the asset - tags: - type: array - items: - type: string - description: Tags associated with the asset - user_metadata: - type: object - description: Custom user metadata - additionalProperties: true - metadata: - type: object - description: System-managed metadata (read-only) - additionalProperties: true - readOnly: true - preview_url: - type: string - format: uri - description: URL for asset preview/thumbnail - preview_id: - type: string - format: uuid - description: ID of the preview asset if available - prompt_id: - type: string - format: uuid - nullable: true - deprecated: true - description: "Deprecated: use job_id instead. ID of the prompt that created this asset." - job_id: - type: string - format: uuid - nullable: true - description: ID of the job that created this asset - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - last_access_time: - type: string - format: date-time - is_immutable: - type: boolean - description: Whether this asset is immutable - - AssetCreated: - description: Response body returned after successfully registering a new asset. - allOf: - - $ref: "#/components/schemas/Asset" - - type: object - required: - - created_new - properties: - created_new: - type: boolean - description: Whether this was a new creation (true) or returned existing (false) - - AssetUpdated: - type: object - description: Response body returned after updating an asset's metadata. - required: - - id - - updated_at - properties: - id: - type: string - format: uuid - name: - type: string - hash: - type: string - nullable: true - description: Blake3 content hash of the asset (preferred over asset_hash) - pattern: "^blake3:[a-f0-9]{64}$" - asset_hash: - type: string - nullable: true - deprecated: true - description: "Deprecated: use `hash` instead. Blake3 hash of the asset content." - pattern: "^blake3:[a-f0-9]{64}$" - tags: - type: array - items: - type: string - mime_type: - type: string - user_metadata: - type: object - additionalProperties: true - prompt_id: - type: string - format: uuid - nullable: true - deprecated: true - description: "Deprecated: use job_id instead. ID of the prompt that created this asset." - job_id: - type: string - format: uuid - nullable: true - description: ID of the job that created this asset - updated_at: - type: string - format: date-time - - ListAssetsResponse: - type: object - description: Paginated list of assets. - required: - - assets - - total - - has_more - properties: - assets: - type: array - items: - $ref: "#/components/schemas/Asset" - total: - type: integer - has_more: - type: boolean - - TagInfo: - type: object - description: A tag known to the asset database, with the number of assets bearing it. - required: - - name - - count - properties: - name: - type: string - count: - type: integer - - ListTagsResponse: - type: object - description: Flat list of all tags, with counts. - required: - - tags - - total - - has_more - properties: - tags: - type: array - items: - $ref: "#/components/schemas/TagInfo" - total: - type: integer - has_more: - type: boolean - - AssetTagHistogramResponse: - type: object - description: Tags that would refine a filtered asset query, with the count of assets each tag would additionally select. - required: - - tag_counts - properties: - tag_counts: - type: object - additionalProperties: - type: integer - description: Map of tag names to occurrence counts - - TagsModificationResponse: - type: object - description: Response body returned after adding or removing tags on an asset. - required: - - total_tags - properties: - added: - type: array - items: - type: string - description: Tags successfully added - removed: - type: array - items: - type: string - description: Tags successfully removed - already_present: - type: array - items: - type: string - description: Tags already present (for add) - not_present: - type: array - items: - type: string - description: Tags not present (for remove) - total_tags: - type: array - items: - type: string - description: All tags on the asset after the operation - - # ------------------------------------------------------------------- - # Result / Output types - # ------------------------------------------------------------------- - ResultItem: - type: object - description: A single output file reference - properties: - filename: - type: string - subfolder: - type: string - type: - type: string - enum: [input, output, temp] - display_name: - type: string - - NodeOutputs: - type: object - description: | - Outputs from a single node execution. Known keys are listed below, - but custom nodes may add arbitrary keys (additionalProperties). - properties: - images: - type: array - items: - $ref: "#/components/schemas/ResultItem" - audio: - type: array - items: - $ref: "#/components/schemas/ResultItem" - video: - type: array - items: - $ref: "#/components/schemas/ResultItem" - animated: - type: array - items: - type: boolean - text: - oneOf: - - type: string - - type: array - items: - type: string - additionalProperties: true - - TerminalSize: - type: object - description: Terminal dimensions - properties: - cols: - type: number - row: - type: number - - LogEntry: - type: object - description: A single log entry - properties: - t: - type: string - description: Timestamp - m: - type: string - description: Log message - - StatusWsMessageStatus: - type: object - description: Inner payload of a `status` WebSocket message, describing the execution queue state. - properties: - exec_info: - type: object - required: - - queue_remaining - properties: - queue_remaining: - type: integer - - StatusWsMessage: - type: object - description: Initial status message sent on connect + queue status updates - properties: - status: - $ref: "#/components/schemas/StatusWsMessageStatus" - sid: - type: string - description: Session ID assigned by the server - - ProgressWsMessage: - type: object - description: Node execution progress (step N of M) - required: - - value - - max - - prompt_id - - node - properties: - value: - type: integer - description: Current step - max: - type: integer - description: Total steps - prompt_id: - type: string - node: - type: string - description: Node ID currently executing - - ProgressTextWsMessage: - type: object - description: Text-based progress update from a node - properties: - nodeId: - type: string - text: - type: string - prompt_id: - type: string - - NodeProgressState: - type: object - description: Progress state for a single node - properties: - value: - type: number - max: - type: number - state: - type: string - enum: [pending, running, finished, error] - node_id: - type: string - prompt_id: - type: string - display_node_id: - type: string - parent_node_id: - type: string - real_node_id: - type: string - - ProgressStateWsMessage: - type: object - description: Bulk progress state for all nodes in a prompt - required: - - prompt_id - - nodes - properties: - prompt_id: - type: string - nodes: - type: object - description: Map of node ID to progress state - additionalProperties: - $ref: "#/components/schemas/NodeProgressState" - - ExecutingWsMessage: - type: object - description: Fired when a node begins execution - required: - - node - - display_node - - prompt_id - properties: - node: - type: string - description: Node ID - display_node: - type: string - description: Display node ID (may differ for subgraphs) - prompt_id: - type: string - - ExecutedWsMessage: - type: object - description: Fired when a node completes execution with output - required: - - node - - display_node - - prompt_id - - output - properties: - node: - type: string - display_node: - type: string - prompt_id: - type: string - output: - $ref: "#/components/schemas/NodeOutputs" - merge: - type: boolean - description: Whether to merge with existing output - - ExecutionWsMessageBase: - type: object - description: Base fields for execution lifecycle messages - required: - - prompt_id - - timestamp - properties: - prompt_id: - type: string - timestamp: - type: integer - description: Unix timestamp in milliseconds - - ExecutionStartWsMessage: - allOf: - - $ref: "#/components/schemas/ExecutionWsMessageBase" - description: Fired when prompt execution begins - - ExecutionSuccessWsMessage: - allOf: - - $ref: "#/components/schemas/ExecutionWsMessageBase" - description: Fired when prompt execution completes successfully - - ExecutionCachedWsMessage: - allOf: - - $ref: "#/components/schemas/ExecutionWsMessageBase" - - type: object - properties: - nodes: - type: array - items: - type: string - description: List of node IDs that were cached - description: Fired when nodes are served from cache - - ExecutionInterruptedWsMessage: - allOf: - - $ref: "#/components/schemas/ExecutionWsMessageBase" - - type: object - properties: - node_id: - type: string - node_type: - type: string - executed: - type: array - items: - type: string - description: Node IDs that completed before interruption - description: Fired when execution is interrupted by user - - ExecutionErrorWsMessage: - allOf: - - $ref: "#/components/schemas/ExecutionWsMessageBase" - - type: object - properties: - node_id: - type: string - node_type: - type: string - executed: - type: array - items: - type: string - exception_message: - type: string - exception_type: - type: string - traceback: - type: array - items: - type: string - current_inputs: {} - current_outputs: {} - description: Fired when a node throws an exception during execution - - LogsWsMessage: - type: object - description: Streaming log entries from the server - properties: - size: - $ref: "#/components/schemas/TerminalSize" - entries: - type: array - items: - $ref: "#/components/schemas/LogEntry" - - NotificationWsMessage: - type: object - description: Server notification (e.g. model download complete) - properties: - value: - type: string - id: - type: string - - FeatureFlagsWsMessage: - type: object - description: Feature flags sent on connect - additionalProperties: true - - AssetDownloadWsMessage: - type: object - description: Asset download progress - required: - - task_id - - asset_name - - bytes_total - - bytes_downloaded - - progress - - status - properties: - task_id: - type: string - asset_name: - type: string - bytes_total: - type: number - bytes_downloaded: - type: number - progress: - type: number - description: 0.0 to 1.0 - status: - type: string - enum: [created, running, completed, failed] - asset_id: - type: string - error: - type: string - - AssetExportWsMessage: - type: object - description: Bulk asset export progress - required: - - task_id - - assets_total - - assets_attempted - - assets_failed - - bytes_total - - bytes_processed - - progress - - status - properties: - task_id: - type: string - export_name: - type: string - assets_total: - type: number - assets_attempted: - type: number - assets_failed: - type: number - bytes_total: - type: number - bytes_processed: - type: number - progress: - type: number - description: 0.0 to 1.0 - status: - type: string - enum: [created, running, completed, failed] - error: - type: string - - # ------------------------------------------------------------------- - # Cloud-runtime schemas - # - # These schemas are exclusively referenced by cloud-runtime operations. - # Tagged x-runtime: [cloud]. - # ------------------------------------------------------------------- - CloudError: - type: object - x-runtime: [cloud] - description: "[cloud-only] Standard error response from cloud endpoints." - required: - - error - properties: - error: - type: string - description: Error message - code: - type: string - description: Machine-readable error code - details: - type: object - additionalProperties: true - description: Additional error context - - CloudJobStatus: - type: object - x-runtime: [cloud] - description: "[cloud-only] Status of a cloud job." - required: - - id - - status - properties: - id: - type: string - format: uuid - status: - type: string - enum: [pending, running, completed, failed, cancelled] - progress: - type: number - minimum: 0 - maximum: 1 - description: "Execution progress (0.0 to 1.0)" - started_at: - type: string - format: date-time - nullable: true - completed_at: - type: string - format: date-time - nullable: true - - CloudPrompt: - type: object - x-runtime: [cloud] - description: "[cloud-only] A cloud-executed prompt record." - required: - - id - - status - properties: - id: - type: string - format: uuid - status: - type: string - workflow: - type: object - additionalProperties: true - outputs: - type: object - additionalProperties: true - created_at: - type: string - format: date-time - completed_at: - type: string - format: date-time - nullable: true - - HistoryV2Response: - type: object - x-runtime: [cloud] - description: "[cloud-only] Paginated execution history in v2 format." - required: - - items - - total - - has_more - properties: - items: - type: array - items: - $ref: "#/components/schemas/HistoryV2Entry" - total: - type: integer - has_more: - type: boolean - - HistoryV2Entry: - type: object - x-runtime: [cloud] - description: "[cloud-only] A single execution history entry in v2 format." - required: - - id - - status - properties: - id: - type: string - format: uuid - status: - type: string - workflow: - type: object - additionalProperties: true - outputs: - type: object - additionalProperties: true - created_at: - type: string - format: date-time - started_at: - type: string - format: date-time - nullable: true - completed_at: - type: string - format: date-time - nullable: true - preview_output: - type: object - additionalProperties: true - - CloudLogsResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Paginated cloud execution logs." - required: - - entries - properties: - entries: - type: array - items: - type: object - properties: - timestamp: - type: string - format: date-time - level: - type: string - enum: [debug, info, warn, error] - message: - type: string - job_id: - type: string - format: uuid - total: - type: integer - has_more: - type: boolean - - AssetDownloadRequest: - type: object - x-runtime: [cloud] - description: "[cloud-only] A single asset to download to the cloud runtime." - required: - - asset_id - properties: - asset_id: - type: string - format: uuid - description: ID of the asset to download - target_path: - type: string - description: Target path on the runtime filesystem - - ImportPublishedAssetsRequest: - type: object - x-runtime: [cloud] - description: "[cloud-only] Request body for importing published assets into the caller's library." - required: - - published_asset_ids - properties: - published_asset_ids: - type: array - description: IDs of published assets (inputs and models) to import. - items: - type: string - share_id: - type: string - nullable: true - description: | - Optional. Share ID of the published workflow these assets belong to. When provided (non-null, non-empty): all `published_asset_ids` must belong to this share's workflow version; returns 400 if the share is not found or any asset does not belong to it. When omitted, null, or empty string: no share-scoped validation is performed and the assets are validated only against global rules (preserved for clients that have not yet adopted `share_id`). - - ImportPublishedAssetsResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Response after importing published assets. Each returned `AssetInfo.id` is the caller's newly-created private asset ID, not the published asset ID supplied in the request." - required: - - assets - properties: - assets: - type: array - items: - $ref: "#/components/schemas/AssetInfo" - - RemoteAssetMetadata: - type: object - x-runtime: [cloud] - description: "[cloud-only] Metadata fetched from a remote asset URL." - properties: - content_type: - type: string - description: MIME type of the remote file - content_length: - type: integer - format: int64 - description: Size in bytes - filename: - type: string - description: Suggested filename from Content-Disposition or URL - - CloudNode: - type: object - x-runtime: [cloud] - description: "[cloud-only] An installed custom node package in the cloud runtime." - required: - - id - - name - properties: - id: - type: string - name: - type: string - version: - type: string - description: - type: string - author: - type: string - repository: - type: string - format: uri - installed_at: - type: string - format: date-time - enabled: - type: boolean - - HubLabel: - type: object - x-runtime: [cloud] - description: "[cloud-only] A label/category used for tagging hub content." - required: - - id - - name - properties: - id: - type: string - name: - type: string - description: - type: string - color: - type: string - description: Hex color code for the label - - HubProfile: - type: object - x-runtime: [cloud] - description: "[cloud-only] A public user profile on the ComfyUI Hub." - required: - - username - properties: - username: - type: string - display_name: - type: string - bio: - type: string - avatar_url: - type: string - format: uri - links: - type: array - items: - type: string - format: uri - workflow_count: - type: integer - created_at: - type: string - format: date-time - - HubWorkflow: - type: object - x-runtime: [cloud] - description: "[cloud-only] A published workflow on the ComfyUI Hub." - required: - - share_id - - name - properties: - share_id: - type: string - name: - type: string - description: - type: string - author: - $ref: "#/components/schemas/HubProfile" - labels: - type: array - items: - $ref: "#/components/schemas/HubLabel" - thumbnail_url: - type: string - format: uri - content: - type: object - additionalProperties: true - description: Workflow graph JSON - likes: - type: integer - views: - type: integer - forks: - type: integer - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - - HubWorkflowList: - type: object - x-runtime: [cloud] - description: "[cloud-only] Paginated list of hub workflows." - required: - - workflows - - total - - has_more - properties: - workflows: - type: array - items: - $ref: "#/components/schemas/HubWorkflow" - total: - type: integer - has_more: - type: boolean - - HubWorkflowIndexEntry: - type: object - x-runtime: [cloud] - description: "[cloud-only] Lightweight entry in the hub workflow index for client-side search." - required: - - share_id - - name - properties: - share_id: - type: string - name: - type: string - author_username: - type: string - labels: - type: array - items: - type: string - likes: - type: integer - updated_at: - type: string - format: date-time - - CloudWorkflow: - type: object - x-runtime: [cloud] - description: "[cloud-only] A cloud-managed workflow with version history." - required: - - id - - name - properties: - id: - type: string - format: uuid - name: - type: string - description: - type: string - share_id: - type: string - nullable: true - description: Public share identifier if published - latest_version_id: - type: string - format: uuid - nullable: true - thumbnail_url: - type: string - format: uri - nullable: true - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - - CloudWorkflowList: - type: object - x-runtime: [cloud] - description: "[cloud-only] Paginated list of cloud workflows." - required: - - workflows - - total - - has_more - properties: - workflows: - type: array - items: - $ref: "#/components/schemas/CloudWorkflow" - total: - type: integer - has_more: - type: boolean - - CloudWorkflowVersion: - type: object - x-runtime: [cloud] - description: "[cloud-only] A version of a cloud workflow." - required: - - id - - workflow_id - properties: - id: - type: string - format: uuid - workflow_id: - type: string - format: uuid - version_number: - type: integer - created_at: - type: string - format: date-time - - AuthSession: - type: object - x-runtime: [cloud] - description: "[cloud-only] Current authentication session state." - required: - - user - properties: - user: - $ref: "#/components/schemas/CloudUser" - workspace: - $ref: "#/components/schemas/Workspace" - expires_at: - type: string - format: date-time - - AuthTokenResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] OAuth2 token response." - required: - - access_token - - token_type - properties: - access_token: - type: string - token_type: - type: string - description: Always "Bearer" - expires_in: - type: integer - description: Token lifetime in seconds - refresh_token: - type: string - nullable: true - scope: - type: string - - JwksResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] JSON Web Key Set for JWT verification." - required: - - keys - properties: - keys: - type: array - items: - type: object + type: string + file_path: + description: Relative path in global-namespace-root form (e.g. "models/checkpoints/flux.safetensors") + nullable: true + type: string + hash: + description: Blake3 hash of the asset content. + pattern: ^blake3:[a-f0-9]{64}$ + type: string + id: + description: Asset ID + format: uuid + type: string + job_id: + description: ID of the job that created this asset, if available + format: uuid + nullable: true + type: string + mime_type: + description: Updated MIME type of the asset + type: string + name: + description: Updated name of the asset + type: string + tags: + description: Tags associated with the asset + items: + type: string + type: array + updated_at: + description: Timestamp of the update + format: date-time + type: string + user_metadata: + additionalProperties: true + description: Updated custom metadata + type: object required: - - kty - - kid - - use + - id + - updated_at + type: object + CreateWorkflowRequest: + description: Request body for creating a new saved workflow. properties: - kty: - type: string - description: Key type (e.g. RSA) - kid: - type: string - description: Key ID - use: - type: string - description: Key use (e.g. sig) - alg: - type: string - description: Algorithm (e.g. RS256) - n: - type: string - description: RSA modulus (base64url) - e: - type: string - description: RSA exponent (base64url) - additionalProperties: true - - OAuthAuthorizationServerMetadata: - type: object - x-runtime: [cloud] - description: "[cloud-only] OAuth 2.1 authorization-server metadata (RFC 8414)." - required: - - issuer - - authorization_endpoint - - token_endpoint - - jwks_uri - - response_types_supported - - grant_types_supported - - code_challenge_methods_supported - - token_endpoint_auth_methods_supported - properties: - issuer: - type: string - format: uri - authorization_endpoint: - type: string - format: uri - token_endpoint: - type: string - format: uri - jwks_uri: - type: string - format: uri - registration_endpoint: - type: string - format: uri - description: "[cloud-only] RFC 7591 §3.1 Dynamic Client Registration endpoint. Advertised so MCP-spec-compliant clients can auto-discover and self-register without operator involvement. Present only when DCR is enabled." - response_types_supported: - type: array - items: - type: string - grant_types_supported: - type: array - items: - type: string - code_challenge_methods_supported: - type: array - items: - type: string - token_endpoint_auth_methods_supported: - type: array - items: - type: string - scopes_supported: - type: array - items: - type: string - - OAuthProtectedResourceMetadata: - type: object - x-runtime: [cloud] - description: "[cloud-only] OAuth 2.1 protected-resource metadata (RFC 9728)." - required: - - resource - - authorization_servers - - scopes_supported - properties: - resource: - type: string - format: uri - authorization_servers: - type: array - items: - type: string - format: uri - scopes_supported: - type: array - items: - type: string - bearer_methods_supported: - type: array - items: - type: string - - OAuthConsentChallenge: - type: object - x-runtime: [cloud] - description: "[cloud-only] Server-side state describing the OAuth consent decision the user is being asked to make. Returned by GET /oauth/authorize when a valid session exists; the frontend renders the consent UI from this payload and POSTs the decision back. Browser never sees the original OAuth params on resume." - required: - - oauth_request_id - - csrf_token - - client_display_name - - resource_display_name - - scopes - - workspaces - properties: - oauth_request_id: - type: string - format: uuid - description: Opaque server-side identifier for the authorization-request row. Carried back unchanged in the consent submission. - csrf_token: - type: string - description: Per-row CSRF token bound to this authorization request (not to the session). Must be echoed back on POST. - client_display_name: - type: string - description: Human-readable name of the OAuth client requesting authorization. - resource_display_name: - type: string - description: Human-readable name of the protected resource. - scopes: - type: array - description: Scopes the client is requesting for this resource. The frontend should present these for the user to approve. - items: - type: string - workspaces: - type: array - description: Workspaces the user can select from. Membership is re-checked on POST. - items: - $ref: "#/components/schemas/OAuthConsentChallengeWorkspace" - - OAuthConsentChallengeWorkspace: - type: object - x-runtime: [cloud] - description: "[cloud-only] One workspace option presented in the OAuth consent challenge." - required: [id, name, type, role] - properties: - id: { type: string } - name: { type: string } - type: { type: string, enum: [personal, team] } - role: { type: string, enum: [owner, member] } - - OAuthAuthorizeRedirectResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Redirect target produced after a JSON consent submission. The frontend must navigate the browser to this URL so custom-scheme client callbacks work without relying on fetch-visible 302 headers." - required: - - redirect_url - properties: - redirect_url: - type: string - format: uri - description: OAuth client redirect URI with either code+state for allow, or error+state for deny. - - OAuthTokenResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] RFC 6749 §5.1 successful token response." - required: [access_token, token_type, expires_in, refresh_token, scope] - properties: - access_token: - type: string - description: Resource-bound access token (audience matches the protected resource). - token_type: - type: string - enum: [Bearer] - expires_in: - type: integer - description: Access token lifetime in seconds. - refresh_token: - type: string - description: Opaque refresh token. Rotates on every successful refresh; presenting an already-rotated token revokes the entire family. - scope: - type: string - description: Space-delimited scopes granted with this token. - - OAuthTokenError: - type: object - x-runtime: [cloud] - description: "[cloud-only] RFC 6749 §5.2 error response." - required: [error] - properties: - error: - type: string - description: 'RFC 6749 §5.2 error code: invalid_request, invalid_client, invalid_grant, unauthorized_client, unsupported_grant_type, invalid_scope.' - error_description: - type: string - description: Human-readable, no leak of internal storage state. - - OAuthRegisterRequest: - type: object - x-runtime: [cloud] - additionalProperties: false - description: "[cloud-only] RFC 7591 §2 client metadata document. Only the fields the server honors are listed; presence of `scope` or `resource_grants` in the request is rejected (`invalid_client_metadata`) because those are server-owned for dynamic clients." - required: - - redirect_uris - - application_type - properties: - redirect_uris: - type: array - items: - type: string - minItems: 1 - maxItems: 5 - description: 1–5 redirect URIs. Validated against `application_type` policy. - client_name: - type: string - maxLength: 100 - description: Human-readable name shown in the consent UI. Reserved-name list rejects impersonation of major clients. - application_type: - type: string - enum: [native, web] - description: | - RFC 7591 §2 application_type. **REQUIRED** — clients MUST declare intent; the server does not default this field. `native` for desktop / CLI / MCP-spec-strict clients (loopback redirects); `web` for hosted clients (HTTPS only, host must be allowlisted). A missing or explicitly empty `application_type` rejects with `invalid_client_metadata`. - token_endpoint_auth_method: - type: string - enum: [none] - description: 'Public clients only this phase — must be `none` if present. The server forces `none` regardless.' - grant_types: - type: array - items: - type: string - enum: [authorization_code, refresh_token] - description: Optional. Defaults to `["authorization_code","refresh_token"]`. - response_types: - type: array - items: - type: string - enum: [code] - description: Optional. Defaults to `["code"]`. - scope: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Dynamic clients do not pick scopes — the server assigns scopes from the active resource's published list. Sending `scope` in the registration body is treated as a privilege-escalation attempt and returns `invalid_client_metadata`." - resource_grants: - type: object - nullable: true - additionalProperties: - type: array - items: - type: string - description: "**REJECTED IF PRESENT.** Same reason as `scope`. The set of resources and scopes a dynamic client may request is server-policy, not request-driven." - client_uri: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - logo_uri: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - tos_uri: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - policy_uri: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - software_id: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - software_version: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - contacts: - type: array - nullable: true - items: - type: string - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - jwks: - type: object - nullable: true - additionalProperties: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - jwks_uri: - type: string - nullable: true - description: "**REJECTED IF PRESENT.** Unsupported RFC 7591 metadata for this public-client phase." - - OAuthRegisterResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] RFC 7591 §3.2.1 successful registration response." - required: - - client_id - - client_id_issued_at - - redirect_uris - - grant_types - - response_types - - token_endpoint_auth_method - - application_type - properties: - client_id: - type: string - description: Server-generated client_id. - client_id_issued_at: - type: integer - format: int64 - description: Unix timestamp (seconds) when the client was registered. - client_name: - type: string - redirect_uris: - type: array - items: - type: string - grant_types: - type: array - items: - type: string - response_types: - type: array - items: - type: string - token_endpoint_auth_method: - type: string - enum: [none] - application_type: - type: string - enum: [native, web] - - OAuthRegisterError: - type: object - x-runtime: [cloud] - description: "[cloud-only] RFC 7591 §3.2.2 error response." - required: - - error - properties: - error: - type: string - enum: [invalid_redirect_uri, invalid_client_metadata] - error_description: - type: string - nullable: true - - BillingBalance: - type: object - x-runtime: [cloud] - description: "[cloud-only] Current credit balance and usage summary." - required: - - credits_remaining - properties: - credits_remaining: - type: integer - description: Available credits - credits_used: - type: integer - description: Credits used in current billing period - credits_total: - type: integer - description: Total credits allocated in current period - - BillingEvent: - type: object - x-runtime: [cloud] - description: "[cloud-only] A billing event (charge, credit, refund)." - required: - - id - - type - - amount - - created_at - properties: - id: - type: string - type: - type: string - enum: [charge, credit, refund, topup, subscription] - amount: - type: integer - description: Amount in credits - description: - type: string - job_id: - type: string - format: uuid - nullable: true - created_at: - type: string - format: date-time - - BillingEventList: - type: object - x-runtime: [cloud] - description: "[cloud-only] Paginated list of billing events." - required: - - events - - total - - has_more - properties: - events: - type: array - items: - $ref: "#/components/schemas/BillingEvent" - total: - type: integer - has_more: - type: boolean - - BillingOp: - type: object - x-runtime: [cloud] - description: "[cloud-only] A billing operation record." - required: - - id - - status - properties: - id: - type: string - status: - type: string - enum: [pending, completed, failed] - type: - type: string - amount: - type: integer - created_at: - type: string - format: date-time - completed_at: - type: string - format: date-time - nullable: true - - BillingPlan: - type: object - x-runtime: [cloud] - description: "[cloud-only] A subscription plan with pricing details." - required: - - id - - name - properties: - id: - type: string - name: - type: string - description: - type: string - credits_per_month: - type: integer - price_cents: - type: integer - description: Monthly price in cents (USD) - currency: - type: string - default: usd - features: - type: array - items: - type: string - description: List of plan features - - BillingStatus: - type: string - x-runtime: [cloud] - description: "[cloud-only] Overall billing/payment lifecycle status." - enum: - - awaiting_payment_method - - pending_payment - - paid - - payment_failed - - inactive - - BillingSubscription: - type: object - x-runtime: [cloud] - description: "[cloud-only] Active subscription details." - required: - - id - - status - - plan_id - properties: - id: - type: string - status: - type: string - enum: [active, cancelled, past_due, trialing] - plan_id: - type: string - plan_name: - type: string - current_period_start: - type: string - format: date-time - current_period_end: - type: string - format: date-time - cancel_at_period_end: - type: boolean - - SubscriptionPreview: - type: object - x-runtime: [cloud] - description: "[cloud-only] Preview of a subscription change including prorations." - properties: - plan_id: - type: string - plan_name: - type: string - amount_due: - type: integer - description: Amount due in cents - proration_amount: - type: integer - description: Proration adjustment in cents - currency: - type: string - next_billing_date: - type: string - format: date-time - - Workspace: - type: object - x-runtime: [cloud] - description: "[cloud-only] A cloud workspace for team collaboration." - required: - - id - - name - properties: - id: - type: string - name: - type: string - type: - type: string - enum: - - personal - - team - description: Workspace type (personal vs. team). - owner_id: - type: string - member_count: - type: integer - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - - WorkspaceMember: - type: object - x-runtime: [cloud] - description: "[cloud-only] A member of a cloud workspace." - required: - - user_id - - role - properties: - user_id: - type: string - email: - type: string - format: email - display_name: - type: string - avatar_url: - type: string - format: uri - role: - type: string - enum: [owner, admin, member] - joined_at: - type: string - format: date-time - - WorkspaceInvite: - type: object - x-runtime: [cloud] - description: "[cloud-only] A pending workspace invitation." - required: - - id - - email - - role - properties: - id: - type: string - email: - type: string - format: email - role: - type: string - enum: [admin, member] - invited_by: - type: string - created_at: - type: string - format: date-time - expires_at: - type: string - format: date-time - - WorkspaceApiKey: - type: object - x-runtime: [cloud] - description: "[cloud-only] A workspace API key (secret value redacted)." - required: - - id - - name - - description - properties: - id: - type: string - name: - type: string - description: - type: string - maxLength: 5000 - description: User-provided description of the key's purpose. Always present in responses; empty string when no description was supplied on create. - prefix: - type: string - description: First few characters of the key for identification - created_at: - type: string - format: date-time - last_used_at: - type: string - format: date-time - nullable: true - created_by: - type: string - - WorkspaceApiKeyCreated: - type: object - x-runtime: [cloud] - description: "[cloud-only] A newly created workspace API key, including the full secret value (shown only once)." - required: - - id - - name - - description - - key - properties: - id: - type: string - name: - type: string - description: - type: string - maxLength: 5000 - description: User-provided description of the key's purpose. Always present in responses; empty string when no description was supplied on create. - key: - type: string - description: Full API key value (only returned on creation) - prefix: - type: string - created_at: - type: string - format: date-time - - CloudUser: - type: object - x-runtime: [cloud] - description: "[cloud-only] A cloud-authenticated user profile." - required: - - id - - email - properties: - id: - type: string - email: - type: string - format: email - display_name: - type: string - avatar_url: - type: string - format: uri - created_at: - type: string - format: date-time - - SecretMeta: - type: object - x-runtime: [cloud] - description: "[cloud-only] Metadata for a stored secret (value is never returned)." - required: - - id - - name - properties: - id: - type: string - name: - type: string - provider: - type: string - description: "[cloud-only] Provider identifier (e.g., huggingface, civitai)." - x-runtime: [cloud] - last_used_at: - type: string - format: date-time - description: "[cloud-only] When the secret was last used for decryption." - x-runtime: [cloud] - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - - UpdateSecretRequest: - type: object - x-runtime: [cloud] - description: "[cloud-only] Request body for updating an existing user secret." - properties: - name: - type: string - description: New name for the secret - secret_value: - type: string - description: New secret value (API key, token, etc.) - - CreateSessionResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Response after creating a session cookie." - required: - - success - properties: - success: - type: boolean - expiresIn: - type: integer - description: Session expiration time in seconds. - - DeleteSessionResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Response after deleting a session cookie." - required: - - success - properties: - success: - type: boolean - - CreateHubProfileRequest: - type: object - x-runtime: [cloud] - description: "[cloud-only] Request body for creating a new Hub profile." - required: - - workspace_id - - username - properties: - workspace_id: - type: string - username: - type: string - description: Unique URL-safe slug. Immutable after creation. - display_name: - type: string - description: - type: string - avatar_token: - type: string - website_urls: - type: array - items: - type: string - - PublishHubWorkflowRequest: - type: object - x-runtime: [cloud] - description: "[cloud-only] Request body for publishing or updating a workflow on the Hub." - required: - - username - - name - - workflow_filename - - asset_ids - properties: - username: - type: string - name: - type: string - workflow_filename: - type: string - asset_ids: - type: array - items: - type: string - description: - type: string - tags: - type: array - items: - type: string - models: - type: array - items: - type: string - custom_nodes: - type: array - items: - type: string - tutorial_url: - type: string - metadata: - type: object - additionalProperties: true - thumbnail_type: - type: string - enum: [image, video, image_comparison] - thumbnail_token_or_url: - type: string - thumbnail_comparison_token_or_url: - type: string - sample_image_tokens_or_urls: - type: array - items: - type: string - - HubWorkflowDetail: - type: object - x-runtime: [cloud] - description: "[cloud-only] Full Hub workflow detail including versions, assets, and statistics." - required: - - share_id - - workflow_id - - name - - workflow_json - - assets - - profile - - status - properties: - share_id: - type: string - workflow_id: - type: string - name: - type: string - status: - type: string - enum: [pending, approved, rejected, deprecated] - description: - type: string - thumbnail_type: - type: string - enum: [image, video, image_comparison] - thumbnail_url: - type: string - thumbnail_comparison_url: - type: string - tutorial_url: - type: string - metadata: - type: object - additionalProperties: true - sample_image_urls: - type: array - items: - type: string - publish_time: - type: string - format: date-time - nullable: true - workflow_json: - type: object - additionalProperties: true - assets: - type: array - items: - $ref: "#/components/schemas/AssetInfo" - profile: - $ref: "#/components/schemas/HubProfile" - - AssetInfo: - type: object - x-runtime: [cloud] - description: "[cloud-only] Lightweight asset reference used in workflow publishing payloads." - required: - - id - - filename - properties: - id: - type: string - filename: - type: string - mime_type: - type: string - size_bytes: - type: integer - format: int64 - - BulkRevokeAPIKeysResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Response after bulk-revoking API keys for a workspace member." - required: - - revoked_count - properties: - revoked_count: - type: integer - minimum: 0 - - CreateWorkflowVersionRequest: - type: object - x-runtime: [cloud] - description: "[cloud-only] Request body for creating a new version of a saved workflow." - required: - - base_version - - workflow_json - properties: - base_version: - type: integer - description: Version number this change is based on (for optimistic concurrency). - workflow_json: - type: object - additionalProperties: true - - WorkflowVersionResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Metadata for a single workflow version." - required: - - id - - version - - latest_version - - created_by - - created_at - properties: - id: - type: string - version: - type: integer - latest_version: - type: integer - created_by: - type: string - created_at: - type: string - format: date-time - - WorkflowPublishInfo: - type: object - x-runtime: [cloud] - description: "[cloud-only] Publishing metadata for a workflow shared to the Hub." - required: - - workflow_id - - share_id - - listed - - assets - properties: - workflow_id: - type: string - share_id: - type: string - publish_time: - type: string - format: date-time - nullable: true - listed: - type: boolean - assets: - type: array - items: - $ref: "#/components/schemas/AssetInfo" - - TaskEntry: - type: object - x-runtime: [cloud] - description: "[cloud-only] Task data for list views." - required: - - id - - task_name - - status - - create_time - properties: - id: - type: string - format: uuid - task_name: - type: string - status: - type: string - enum: [created, running, completed, failed] - create_time: - type: string - format: date-time - started_at: - type: string - format: date-time - completed_at: - type: string - format: date-time - - TaskResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Full task details including payload and result." - required: - - id - - idempotency_key - - task_name - - payload - - status - - create_time - - update_time - properties: - id: - type: string - format: uuid - idempotency_key: - type: string - task_name: - type: string - payload: - type: object - additionalProperties: true - status: - type: string - enum: [created, running, completed, failed] - result: - type: object - additionalProperties: true - create_time: - type: string - format: date-time - update_time: - type: string - format: date-time - started_at: - type: string - format: date-time - completed_at: - type: string - format: date-time - error: - type: string - - TasksListResponse: - type: object - x-runtime: [cloud] - description: "[cloud-only] Paginated list of background tasks for the authenticated user." - required: - - tasks - - pagination - properties: - tasks: - type: array - items: - $ref: "#/components/schemas/TaskEntry" - pagination: - $ref: "#/components/schemas/PaginationInfo" - - # ===== Cloud-only schemas (Comfy-Org/cloud runtime, BE-1106) ===== - AssetDownloadResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Acknowledgement of an async asset download task; clients poll GET /api/tasks/{task_id} for status.' - required: - - task_id - - status - properties: - task_id: - type: string - format: uuid - description: Task ID for tracking download progress via GET /api/tasks/{task_id} - status: - type: string - enum: - - created - - running - - completed - - failed - description: Current task status - message: - type: string - description: Human-readable message - example: Download task created. Use task_id to track progress. - - AssetMetadataResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Metadata for a remotely hosted asset resolved by URL.' - required: - - content_length - properties: - content_length: - type: integer - format: int64 - description: Size of the asset in bytes (-1 if unknown) - example: 4294967296 - content_type: - type: string - description: MIME type of the asset - example: application/octet-stream - filename: - type: string - description: Suggested filename for the asset from source - example: realistic-vision-v5.safetensors - name: - type: string - description: Display name or title for the asset from source - example: Realistic Vision v5.0 - tags: - type: array - items: - type: string - description: Tags for categorization from source - example: - - models - - checkpoint - preview_image: - type: string - description: Preview image as base64-encoded data URL - example: data:image/jpeg;base64,/9j/4AAQSkZJRg... - validation: - description: Validation results for the file - allOf: - - $ref: '#/components/schemas/ValidationResult' - - BillingBalanceResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Current credit balance and usage details for a workspace.' - required: - - amount_micros - - currency - properties: - amount_micros: - type: number - format: double - description: The total remaining balance in microamount (1/1,000,000 of the currency unit) - prepaid_balance_micros: - type: number - format: double - description: The remaining balance from prepaid commits in microamount - cloud_credit_balance_micros: - type: number - format: double - description: The remaining balance from cloud credits in microamount - pending_charges_micros: - type: number - format: double - description: The total amount of pending/unbilled charges from draft invoices in microamount - effective_balance_micros: - type: number - format: double - description: The effective balance (total balance minus pending charges). Can be negative if pending charges exceed - the balance. - currency: - type: string - example: usd - description: Currency code - - BillingPlansResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] List of available billing plans for subscription.' - required: - - plans - properties: - current_plan_slug: - type: string - description: Current plan slug if subscribed - plans: - type: array - items: - $ref: '#/components/schemas/Plan' - - BillingStatusResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Current billing and subscription status for a workspace.' - required: - - is_active - - has_funds - properties: - is_active: - type: boolean - description: Whether the workspace has an active subscription - subscription_status: - type: string - enum: - - active - - ended - - canceled - description: Subscription activity status (scheduled subscriptions are not returned) - subscription_tier: - $ref: '#/components/schemas/SubscriptionTier' - subscription_duration: - $ref: '#/components/schemas/SubscriptionDuration' - plan_slug: - type: string - description: Plan identifier (e.g., standard-monthly, team-pro-annual) - billing_status: - $ref: '#/components/schemas/BillingStatus' - has_funds: - type: boolean - description: Whether the workspace has available credits - cancel_at: - type: string - format: date-time - description: When the subscription will become inactive (if canceled) - renewal_date: - type: string - format: date-time - description: When the current billing period ends and the next one begins - - GetUserDataResponseFull: - type: array - x-runtime: [cloud] - description: '[cloud-only] List of user data file entries (each with path, size, and modification time) returned when full_info=true.' - items: - $ref: '#/components/schemas/GetUserDataResponseFullFile' - - HistoryDetailEntry: - type: object - x-runtime: [cloud] - description: '[cloud-only] History entry with full prompt data' - properties: - prompt: - type: object - description: Full prompt execution data - properties: - priority: - type: number - format: double - description: Execution priority - prompt_id: - type: string - description: The prompt ID - prompt: - type: object - description: The workflow nodes - additionalProperties: true - extra_data: - type: object - description: Additional execution data - additionalProperties: true - outputs_to_execute: - type: array - items: - type: string - description: Output nodes to execute - outputs: - type: object - description: Output data from execution (generated images, files, etc.) - additionalProperties: true - status: - type: object - description: Execution status and timeline information - additionalProperties: true - meta: - type: object - description: Metadata about the execution and nodes - additionalProperties: true - - HistoryDetailResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Detailed execution history response for a specific prompt. - - Returns a dictionary with prompt_id as key and full history data as value. - - ' - additionalProperties: - $ref: '#/components/schemas/HistoryDetailEntry' - - HistoryResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Execution history response with history array. - - Returns an object with a "history" key containing an array of history entries. - - Each entry includes prompt_id as a property along with execution data. - - ' - required: - - history - properties: - history: - type: array - description: Array of history entries ordered by creation time (newest first) - items: - $ref: '#/components/schemas/HistoryEntry' - - HubLabelInfo: - type: object - x-runtime: [cloud] - description: '[cloud-only] Metadata for a single Hub label.' - required: - - name - - display_name - - type - properties: - name: - type: string - description: Slug identifier. - display_name: - type: string - description: Human-readable display name. - description: - type: string - description: Optional description of the label. - type: - type: string - enum: - - tag - - model - - custom_node - description: Label category. - - HubLabelListResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Response wrapper for the available Hub label catalog.' - required: - - labels - properties: - labels: - type: array - items: - $ref: '#/components/schemas/HubLabelInfo' - description: Available labels, optionally filtered by type. - - HubProfileSummary: - type: object - x-runtime: [cloud] - description: '[cloud-only] Abbreviated Hub profile used in workflow listings.' - required: - - username - properties: - username: - type: string - display_name: - type: string - avatar_url: - type: string - description: Public URL of the profile avatar image. - - HubWorkflowListResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Paginated list of Hub workflows matching search criteria.' - required: - - workflows - properties: - workflows: - type: array - items: - anyOf: - - $ref: '#/components/schemas/HubWorkflowSummary' - - $ref: '#/components/schemas/HubWorkflowDetail' - description: Array of HubWorkflowSummary (default) or HubWorkflowDetail (when detail=true). - next_cursor: - type: string - description: Cursor for the next page, empty if no more results. - - HubWorkflowStatus: - type: string - x-runtime: [cloud] - description: '[cloud-only] Public workflow status. NULL in the database is represented as pending in API responses.' - enum: - - pending - - approved - - rejected - - deprecated - - HubWorkflowSummary: - type: object - x-runtime: [cloud] - description: '[cloud-only] Abbreviated Hub workflow metadata used in search and listing results.' - required: - - share_id - - name - - profile - - status - properties: - share_id: - type: string - name: - type: string - status: - $ref: '#/components/schemas/HubWorkflowStatus' - description: - type: string - tags: - type: array - items: - $ref: '#/components/schemas/LabelRef' - models: - type: array - items: - $ref: '#/components/schemas/LabelRef' - custom_nodes: - type: array - items: - $ref: '#/components/schemas/LabelRef' - thumbnail_type: - type: string - enum: - - image - - video - - image_comparison - thumbnail_url: - type: string - thumbnail_comparison_url: - type: string - publish_time: - type: string - format: date-time - nullable: true - profile: - $ref: '#/components/schemas/HubProfileSummary' - metadata: - type: object - additionalProperties: true - tutorial_url: - type: string - sample_image_urls: - type: array - items: - type: string - - HubWorkflowTemplateEntry: - type: object - x-runtime: [cloud] - description: '[cloud-only] Entry in the curated workflow template gallery shown on the home page.' - required: - - name - - title - - status - properties: - name: - type: string - description: Slug identifier for the template - title: - type: string - status: - $ref: '#/components/schemas/HubWorkflowStatus' - description: - type: string - tags: - type: array - items: - type: string - models: - type: array - items: - type: string - requiresCustomNodes: - type: array - items: - type: string - thumbnailVariant: - type: string - mediaType: - type: string - mediaSubtype: - type: string - size: - type: integer - format: int64 - description: Workflow asset size in bytes. - vram: - type: integer - format: int64 - description: Approximate VRAM requirement in bytes. - usage: - type: integer - format: int64 - description: Usage count reported upstream. - searchRank: - type: integer - format: int64 - description: Search ranking score reported upstream. - isEssential: - type: boolean - description: Whether the template belongs to a module marked as essential. - openSource: - type: boolean - profile: - $ref: '#/components/schemas/HubProfileSummary' - tutorialUrl: - type: string - logos: - type: array - items: - type: object - additionalProperties: true - date: - type: string - description: Publication date in YYYY-MM-DD format - io: - type: object - properties: - inputs: - type: array - items: - type: object - additionalProperties: true - outputs: - type: array - items: - type: object - additionalProperties: true - includeOnDistributions: - type: array - items: - type: string - thumbnailUrl: - type: string - description: Public URL of the primary thumbnail - thumbnailComparisonUrl: - type: string - description: Public URL of the comparison thumbnail - shareId: - type: string - description: Share ID for linking to the hub workflow detail - extendedDescription: - type: string - description: AI-generated extended description of the workflow - metaDescription: - type: string - description: AI-generated SEO meta description (under 160 chars) - howToUse: - type: array - items: - type: string - description: AI-generated step-by-step usage instructions - suggestedUseCases: - type: array - items: - type: string - description: AI-generated suggested use cases - faqItems: - type: array - items: - type: object + default_view: + description: Default view mode + enum: + - workflow + - app + type: string + description: + description: Description of the workflow + type: string + forked_from_workflow_id: + description: ID of the source workflow if forked + type: string + forked_from_workflow_version_id: + description: ID of the source workflow version if forked + type: string + name: + description: Display name for the workflow + type: string + workflow_json: + additionalProperties: true + description: The ComfyUI workflow JSON + type: object required: - - question - - answer - properties: - question: - type: string - answer: - type: string - description: AI-generated FAQ items - contentTemplate: - type: string - description: Content template used for generation (tutorial, showcase, comparison, breakthrough) - - JobStatusResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Job status information' - properties: - id: - type: string - format: uuid - description: The job ID - status: - type: string - enum: - - waiting_to_dispatch - - pending - - in_progress - - completed - - error - - cancelled - description: Current job status - created_at: - type: string - format: date-time - description: When the job was created - updated_at: - type: string - format: date-time - description: When the job was last updated - last_state_update: - type: string - format: date-time - description: When the job status was last changed - assigned_inference: - type: string - nullable: true - description: The inference instance assigned to this job (if any) - error_message: - type: string - nullable: true - description: Error message if the job failed - required: - - id - - status - - created_at - - updated_at - - JobsListResponse: - type: object - x-runtime: [cloud] - description: '[cloud-only] Paginated list of jobs for the authenticated user.' - required: - - jobs - - pagination - properties: - jobs: - type: array - description: Array of jobs ordered by specified sort field - items: - $ref: '#/components/schemas/JobEntry' - pagination: - $ref: '#/components/schemas/PaginationInfo' - - LabelRef: - type: object - x-runtime: [cloud] - description: '[cloud-only] Reference to a Hub label by ID.' - required: - - name - - display_name - properties: - name: - type: string - description: Slug identifier (e.g. "video-generation", "flux"). - display_name: - type: string - description: Human-readable display name (e.g. "Video Generation", "Flux"). - - LogsResponse: - type: array - x-runtime: [cloud] - description: '[cloud-only] System logs response' - items: - type: object - properties: - timestamp: - type: string - format: date-time - description: When the log entry was created - level: - type: string - enum: - - debug - - info - - warn - - error - description: Log level - message: - type: string - description: Log message - source: - type: string - description: Source of the log entry - metadata: + - workflow_json type: object + CreateWorkflowVersionRequest: + description: Request body for creating a new version of a saved workflow. + properties: + base_version: + description: The version number this change is based on (for optimistic concurrency) + type: integer + workflow_json: + additionalProperties: true + description: The updated ComfyUI workflow JSON + type: object + required: + - base_version + - workflow_json + type: object + ErrorResponse: + description: Standard error response with a machine-readable code and human-readable message. + properties: + code: + type: string + details: + additionalProperties: true + description: Optional open object carrying structured, machine-readable context about the error (e.g. offending field names, validation specifics). Absent for most errors; consumers must not assume any particular shape. + type: object + message: + type: string + required: + - code + - message + type: object + ExecutionError: + description: Detailed execution error information from ComfyUI + properties: + current_inputs: + additionalProperties: true + description: Input values at time of failure (empty object if not available) + type: object + current_outputs: + additionalProperties: true + description: Output values at time of failure (empty object if not available) + type: object + exception_message: + description: Human-readable error message + type: string + exception_type: + description: Python exception type (e.g., "RuntimeError") + type: string + node_id: + description: ID of the node that failed + type: string + node_type: + description: Type name of the node (e.g., "KSampler") + type: string + traceback: + description: Array of traceback lines (empty array if not available) + items: + type: string + type: array + required: + - node_id + - node_type + - exception_message + - exception_type + - traceback + - current_inputs + - current_outputs + type: object + FeedbackRequest: + description: Request to submit user feedback + properties: + content: + description: The feedback content or message + type: string + metadata: + additionalProperties: true + description: Additional metadata about the feedback + type: object + rating: + description: User's rating of ComfyUI Cloud experience (1-5 stars) + maximum: 5 + minimum: 1 + type: integer + type: + description: Type of feedback being submitted + enum: + - missing_nodes + - general + - missing_models + type: string + required: + - type + type: object + FeedbackResponse: + description: Response after submitting feedback + type: object + ForkWorkflowRequest: + description: Request body for forking an existing workflow into the user's account. + properties: + name: + description: Name for the forked workflow + type: string + source_version: + description: Version number to fork from + type: integer + required: + - source_version + type: object + GetUserDataResponseFull: + description: List of user data file entries (each with path, size, and modification time) returned when full_info=true. + items: + $ref: '#/components/schemas/GetUserDataResponseFullFile' + type: array + GetUserDataResponseFullFile: + description: Individual file entry within a full user data response. + properties: + modified: + description: UNIX timestamp of the last modification in milliseconds. + format: int64 + type: integer + path: + description: File name or path relative to the user directory. + type: string + size: + description: File size in bytes. + type: integer + type: object + GlobalSubgraphData: + description: Full data for a global subgraph blueprint + properties: + data: + description: The full subgraph JSON data as a string + type: string + info: + description: Additional information about the subgraph + properties: + node_pack: + description: The node pack/module that provides this subgraph + type: string + required: + - node_pack + type: object + name: + description: Display name of the subgraph blueprint + type: string + source: + description: Source type of the subgraph - "templates" for workflow templates or "custom_node" for custom node subgraphs + type: string + required: + - source + - name + - info + - data + type: object + GlobalSubgraphInfo: + description: Metadata for a global subgraph blueprint (without full data) + properties: + data: + description: The full subgraph JSON data (may be empty in list view) + type: string + info: + description: Additional information about the subgraph + properties: + node_pack: + description: The node pack/module that provides this subgraph + type: string + required: + - node_pack + type: object + name: + description: Display name of the subgraph blueprint + type: string + source: + description: Source type of the subgraph - "templates" for workflow templates or "custom_node" for custom node subgraphs + type: string + required: + - source + - name + - info + type: object + HistoryDetailEntry: + description: History entry with full prompt data + properties: + meta: + additionalProperties: true + description: Metadata about the execution and nodes + type: object + outputs: + additionalProperties: true + description: Output data from execution (generated images, files, etc.) + type: object + prompt: + description: Full prompt execution data + properties: + extra_data: + additionalProperties: true + description: Additional execution data + type: object + outputs_to_execute: + description: Output nodes to execute + items: + type: string + type: array + priority: + description: Execution priority + format: double + type: number + prompt: + additionalProperties: true + description: The workflow nodes + type: object + prompt_id: + description: The prompt ID + type: string + type: object + status: + additionalProperties: true + description: Execution status and timeline information + type: object + type: object + HistoryDetailResponse: + additionalProperties: + $ref: '#/components/schemas/HistoryDetailEntry' + description: | + Detailed execution history response for a specific prompt. + Returns a dictionary with prompt_id as key and full history data as value. + type: object + HistoryEntry: + description: History entry with prompt_id and execution data + properties: + create_time: + description: Job creation timestamp (Unix timestamp in milliseconds) + format: int64 + type: integer + meta: + additionalProperties: true + description: Metadata about the execution and nodes + type: object + outputs: + additionalProperties: true + description: Output data from execution (generated images, files, etc.) + type: object + prompt: + description: Filtered prompt execution data (lightweight format) + properties: + extra_data: + additionalProperties: true + description: Additional execution data (workflow removed from extra_pnginfo) + type: object + priority: + description: Execution priority + format: double + type: number + prompt_id: + description: The prompt ID + type: string + type: object + prompt_id: + description: Unique identifier for this prompt execution + type: string + status: + additionalProperties: true + description: Execution status and timeline information + type: object + workflow_id: + description: UUID identifying the workflow graph definition + type: string + required: + - prompt_id + type: object + HistoryManageRequest: + additionalProperties: false + description: Request to manage history operations + properties: + clear: + description: If true, clear all history for the authenticated user + type: boolean + delete: + description: Array of job IDs to delete from history + items: + type: string + type: array + type: object + HistoryResponse: + description: | + Execution history response with history array. + Returns an object with a "history" key containing an array of history entries. + Each entry includes prompt_id as a property along with execution data. + properties: + history: + description: Array of history entries ordered by creation time (newest first) + items: + $ref: '#/components/schemas/HistoryEntry' + type: array + required: + - history + type: object + JobCancelResponse: + description: Response for POST /api/jobs/{job_id}/cancel. Returned on both fresh cancels and idempotent no-ops. + properties: + cancelled: + description: | + True when a cancel event was successfully dispatched by this call. + False when the job was already in a terminal or cancelling state, + in which case the call is a no-op (still 200 — idempotent). + type: boolean + required: + - cancelled + type: object + JobDetailResponse: + description: Full job details including workflow and outputs + properties: + create_time: + description: Job creation timestamp (Unix timestamp in milliseconds) + format: int64 + type: integer + execution_error: + allOf: + - $ref: '#/components/schemas/ExecutionError' + description: Detailed execution error from ComfyUI (only for failed jobs with structured error data) + execution_meta: + additionalProperties: true + description: Node-level execution metadata (only for terminal states) + type: object + execution_status: + additionalProperties: true + description: ComfyUI execution status and timeline (only for terminal states) + type: object + id: + description: Unique job identifier + format: uuid + type: string + outputs: + additionalProperties: true + description: Full outputs object from ComfyUI (only for terminal states) + type: object + outputs_count: + description: Total number of output files (omitted for non-terminal states) + type: integer + preview_output: + additionalProperties: true + description: Primary preview output (only for terminal states) + type: object + status: + description: User-friendly job status + enum: + - pending + - in_progress + - completed + - failed + - cancelled + type: string + update_time: + description: Last update timestamp (Unix timestamp in milliseconds) + format: int64 + type: integer + workflow: + additionalProperties: true + description: | + Full ComfyUI workflow (10-100KB, omitted if not available). + + Sensitive credentials are redacted before the response is returned: + `extra_data.api_key_comfy_org`, when present, is replaced with the + literal string `"[REDACTED]"`. The field is preserved (not removed) + so existence checks still pass, but the value is not usable. + type: object + workflow_id: + description: UUID identifying the workflow graph definition + type: string + required: + - id + - status + - create_time + - update_time + type: object + JobEntry: + description: Lightweight job data for list views (workflow and full outputs excluded) + properties: + create_time: + description: Job creation timestamp (Unix timestamp in milliseconds) + format: int64 + type: integer + execution_end_time: + description: Workflow execution completion timestamp (Unix milliseconds, only present for terminal states) + format: int64 + type: integer + execution_error: + allOf: + - $ref: '#/components/schemas/ExecutionError' + description: Detailed execution error from ComfyUI (only for failed jobs with structured error data) + execution_start_time: + description: Workflow execution start timestamp (Unix milliseconds, only present for terminal states) + format: int64 + type: integer + id: + description: Unique job identifier + format: uuid + type: string + outputs_count: + description: Total number of output files (omitted for non-terminal states) + type: integer + preview_output: + additionalProperties: true + description: Primary preview output (only present for terminal states) + type: object + status: + description: User-friendly job status + enum: + - pending + - in_progress + - completed + - failed + - cancelled + type: string + workflow_id: + description: UUID identifying the workflow graph definition + type: string + required: + - id + - status + - create_time + type: object + JobStatusResponse: + description: Job status information + properties: + assigned_inference: + description: The inference instance assigned to this job (if any) + nullable: true + type: string + created_at: + description: When the job was created + format: date-time + type: string + error_message: + description: Error message if the job failed + nullable: true + type: string + id: + description: The job ID + format: uuid + type: string + last_state_update: + description: When the job status was last changed + format: date-time + type: string + status: + description: Current job status + enum: + - waiting_to_dispatch + - pending + - in_progress + - completed + - error + - cancelled + type: string + updated_at: + description: When the job was last updated + format: date-time + type: string + required: + - id + - status + - created_at + - updated_at + type: object + JobsCancelRequest: + additionalProperties: false + description: Request to cancel multiple jobs by ID. + properties: + job_ids: + description: Job identifiers (UUIDs) to cancel. + items: + format: uuid + type: string + maxItems: 100 + minItems: 1 + type: array + required: + - job_ids + type: object + JobsCancelResponse: + description: Response for POST /api/jobs/cancel. + properties: + cancelled: + description: | + Job IDs for which a cancel event was successfully dispatched by this + call. Jobs already in a terminal or cancelling state are idempotently + skipped and will not appear here. + items: + type: string + type: array + required: + - cancelled + type: object + JobsListResponse: + description: Paginated list of jobs for the authenticated user. + properties: + jobs: + description: Array of jobs ordered by specified sort field + items: + $ref: '#/components/schemas/JobEntry' + type: array + pagination: + $ref: '#/components/schemas/PaginationInfo' + required: + - jobs + - pagination + type: object + ListAssetsResponse: + description: Paginated list of assets belonging to the authenticated user. + properties: + assets: + description: List of assets matching the query + items: + $ref: '#/components/schemas/Asset' + type: array + has_more: + description: Whether more assets are available beyond this page + type: boolean + next_cursor: + description: | + Opaque cursor to pass as the `after` query parameter to fetch the + next page. Omitted from the response when there are no more results. + type: string + total: + description: Total number of assets matching the filters + type: integer + required: + - assets + - total + - has_more + type: object + ListTagsResponse: + description: Paginated list of available asset tags. + properties: + has_more: + description: Whether more tags are available + type: boolean + tags: + description: List of tags + items: + $ref: '#/components/schemas/TagInfo' + type: array + total: + description: Total number of tags + type: integer + required: + - tags + - total + - has_more + type: object + ModelFile: + description: Represents a model file with metadata + properties: + name: + description: The filename of the model + example: model.safetensors + type: string + pathIndex: + description: Index of the path where this model is located + example: 0 + type: integer + required: + - name + - pathIndex + type: object + ModelFolder: + description: Represents a folder containing models + properties: + folders: + description: List of paths where models of this type are stored + example: + - checkpoints + items: + type: string + type: array + name: + description: The name of the model folder + example: checkpoints + type: string + required: + - name + - folders + type: object + NodeInfo: + description: Metadata describing a single ComfyUI node type and its inputs/outputs. + properties: + api_node: + description: Whether this is an API node + type: boolean + category: + description: Category of the node + type: string + deprecated: + description: Whether the node is deprecated + type: boolean + description: + description: Description of the node + type: string + display_name: + description: Display name of the node + type: string + experimental: + description: Whether the node is experimental + type: boolean + input: + additionalProperties: true + description: Input specifications for the node + type: object + input_order: + additionalProperties: + items: + type: string + type: array + description: Order of inputs for display + type: object + name: + description: Internal name of the node + type: string + output: + description: Output types of the node + items: + type: string + type: array + output_is_list: + description: Whether each output is a list + items: + type: boolean + type: array + output_name: + description: Names of the outputs + items: + type: string + type: array + output_node: + description: Whether this is an output node + type: boolean + output_tooltips: + description: Tooltips for outputs + items: + type: string + type: array + python_module: + description: Python module implementing the node + type: string + type: object + PaginationInfo: + description: | + Pagination metadata included in list responses. Supports both legacy + offset/limit pagination and cursor-based pagination. When cursor-based + pagination is used, `next_cursor` is the primary pagination token and + `offset`/`total` may be zero. + properties: + has_more: + description: Whether more items are available beyond this page + type: boolean + limit: + description: Items per page + minimum: 1 + type: integer + next_cursor: + description: | + Opaque cursor for the next page. Pass this value as the `after` + query parameter on the next request. Empty or absent when there + are no more results. + type: string + offset: + deprecated: true + description: 'Current offset (0-based). Deprecated: use cursor-based pagination.' + minimum: 0 + type: integer + total: + description: Total number of items matching filters (may be 0 when using cursor pagination) + minimum: 0 + type: integer + required: + - offset + - limit + - total + - has_more + type: object + PromptErrorResponse: additionalProperties: true - description: Additional log metadata + description: Error response for ComfyUI prompt execution. + type: object + PromptInfo: + description: Metadata about the currently running and queued prompts. + properties: + exec_info: + properties: + queue_remaining: + description: Number of items remaining in the queue + type: integer + type: object + type: object + PromptRequest: + description: Request body for submitting a ComfyUI workflow prompt for execution. + properties: + extra_data: + additionalProperties: true + description: Extra data to be associated with the prompt + type: object + front: + description: If true, adds the prompt to the front of the queue + type: boolean + number: + description: Priority number for the queue (lower numbers have higher priority) + type: number + partial_execution_targets: + description: List of node names to execute + items: + type: string + type: array + prompt: + additionalProperties: true + description: The workflow graph to execute + type: object + workflow_id: + description: UUID identifying the cloud workflow entity to associate with this job + type: string + workflow_version_id: + description: UUID identifying the workflow version to associate with this job + type: string + required: + - prompt + type: object + PromptResponse: + description: Response returned after successfully queuing a workflow prompt. + properties: + node_errors: + additionalProperties: true + description: Any errors in the nodes of the prompt + type: object + number: + description: Priority number in the queue + type: number + prompt_id: + description: Unique identifier for the prompt execution + format: uuid + type: string + type: object + PublishWorkflowAssetsRequest: + description: Request body for publishing workflow assets to the Hub. + properties: + asset_ids: + description: IDs of assets (inputs and models) to snapshot. + items: + type: string + type: array + required: + - asset_ids + type: object + PublishedWorkflowDetail: + description: Full detail of a publicly published workflow on the Hub. + properties: + assets: + description: Published assets with their library status for the caller. + items: + $ref: '#/components/schemas/AssetInfo' + type: array + listed: + type: boolean + name: + description: Human-readable workflow name. + type: string + publish_time: + format: date-time + nullable: true + type: string + share_id: + type: string + workflow_id: + type: string + workflow_json: + additionalProperties: true + description: The workflow JSON content at publish time. + type: object + required: + - share_id + - workflow_id + - name + - listed + - workflow_json + - assets + type: object + QueueInfo: + description: Queue information with pending and running jobs + properties: + queue_pending: + description: Array of pending job items (ordered by creation time, oldest first) + items: + description: | + Queue item tuple format: [job_number, prompt_id, workflow_json, output_node_ids, metadata] + - [0] job_number (integer): Position in queue (1-based) + - [1] prompt_id (string): Job UUID + - [2] workflow_json (object): Full ComfyUI workflow + - [3] output_node_ids (array): Node IDs to return results from + - [4] metadata (object): Contains {create_time: } + items: {} + maxItems: 5 + minItems: 5 + type: array + type: array + queue_running: + description: Array of currently running job items + items: + description: | + Queue item tuple format: [job_number, prompt_id, workflow_json, output_node_ids, metadata] + - [0] job_number (integer): Position in queue (1-based) + - [1] prompt_id (string): Job UUID + - [2] workflow_json (object): Full ComfyUI workflow + - [3] output_node_ids (array): Node IDs to return results from + - [4] metadata (object): Contains {create_time: } + items: {} + maxItems: 5 + minItems: 5 + type: array + type: array + type: object + QueueManageRequest: + additionalProperties: false + description: Request to manage queue operations + properties: + clear: + description: If true, clear all pending jobs from the queue + type: boolean + delete: + description: Array of job IDs to cancel; pending and running jobs transition to cancelled + items: + type: string + type: array + type: object + QueueManageResponse: + description: Response after a queue management action (delete or clear). + properties: + cleared: + description: Whether the queue was cleared + type: boolean + deleted: + description: Array of job IDs that were successfully cancelled + items: + type: string + type: array + type: object + SystemStatsResponse: + description: System statistics response + properties: + devices: + items: + properties: + name: + description: Device name + type: string + type: + description: Device type + type: string + vram_free: + description: Free VRAM in bytes + type: number + vram_total: + description: Total VRAM in bytes + type: number + required: + - name + - type + type: object + type: array + system: + properties: + argv: + description: Command line arguments + items: + type: string + type: array + cloud_version: + description: Cloud ingest service version (commit hash) + type: string + comfyui_frontend_version: + description: ComfyUI frontend version (commit hash or tag) + type: string + comfyui_version: + description: ComfyUI version + type: string + deploy_environment: + description: How this ComfyUI instance is deployed (e.g. cloud, local-git, local-portable, local-desktop) + type: string + embedded_python: + description: Whether using embedded Python + type: boolean + os: + description: Operating system + type: string + python_version: + description: Python version + type: string + pytorch_version: + description: PyTorch version + type: string + ram_free: + description: Free RAM in bytes + type: number + ram_total: + description: Total RAM in bytes + type: number + workflow_templates_version: + description: Workflow templates version + type: string + required: + - os + - python_version + - embedded_python + - comfyui_version + - pytorch_version + - argv + - ram_total + - ram_free + type: object + required: + - system + - devices + type: object + TagInfo: + description: Metadata for a single tag that can be applied to assets. + properties: + count: + description: Number of assets using this tag + type: integer + name: + description: Tag name + type: string + required: + - name + - count + type: object + TagsModificationResponse: + description: Response after adding, updating, or removing tags on an asset. + properties: + added: + description: Tags that were successfully added (for add operation) + items: + type: string + type: array + already_present: + description: Tags that were already present (for add operation) + items: + type: string + type: array + not_present: + description: Tags that were not present (for remove operation) + items: + type: string + type: array + removed: + description: Tags that were successfully removed (for remove operation) + items: + type: string + type: array + total_tags: + description: All tags on the asset after the operation + items: + type: string + type: array + required: + - total_tags + type: object + TaskEntry: + description: Task data for list views + properties: + completed_at: + description: When task completed or failed (null if not finished) + format: date-time + type: string + create_time: + description: Task creation timestamp + format: date-time + type: string + id: + description: Unique task identifier + format: uuid + type: string + started_at: + description: When task execution started (null if not started) + format: date-time + type: string + status: + description: Current task status + enum: + - created + - running + - completed + - failed + type: string + task_name: + description: Task type name (e.g., model_upload) + type: string + required: + - id + - task_name + - status + - create_time + type: object + TaskResponse: + description: Full task details including payload and result + properties: + completed_at: + description: When task completed or failed (null if not finished) + format: date-time + type: string + create_time: + description: Task creation timestamp + format: date-time + type: string + error_message: + description: Error message on failure (null if not failed) + type: string + id: + description: Unique task identifier + format: uuid + type: string + idempotency_key: + description: Caller-provided key for idempotent task creation + type: string + payload: + additionalProperties: true + description: Task input data + type: object + result: + additionalProperties: true + description: Task output data (null if not completed) + type: object + started_at: + description: When task execution started (null if not started) + format: date-time + type: string + status: + description: Current task status + enum: + - created + - running + - completed + - failed + type: string + task_name: + description: Task type name (e.g., model_upload) + type: string + update_time: + description: Task last update timestamp + format: date-time + type: string + required: + - id + - idempotency_key + - task_name + - payload + - status + - create_time + - update_time + type: object + TasksListResponse: + description: Paginated list of background tasks for the authenticated user. + properties: + pagination: + $ref: '#/components/schemas/PaginationInfo' + tasks: + description: Array of tasks ordered by create_time + items: + $ref: '#/components/schemas/TaskEntry' + type: array + required: + - tasks + - pagination + type: object + UpdateWorkflowRequest: + description: Request body for updating an existing saved workflow. + properties: + default_view: + description: New default view mode + enum: + - workflow + - app + type: string + description: + description: New description + type: string + name: + description: New display name + type: string + type: object + UserDataResponseFull: + description: User data listing entry with file metadata (path, size, modification time). + properties: + modified: + description: UNIX timestamp of the last modification in milliseconds. + format: int64 + type: integer + path: + type: string + size: + type: integer + type: object + UserResponse: + description: User information response + properties: + id: + description: Firebase UID of the authenticated user + type: string + status: + description: User status (always "active" for authenticated users) + type: string + required: + - id + - status + type: object + WorkflowForkedFrom: + description: Reference to the parent workflow from which this workflow was forked. + properties: + workflow_id: + type: string + workflow_version_id: + type: string + type: object + WorkflowListResponse: + description: Paginated list of saved workflows. + properties: + data: + items: + $ref: '#/components/schemas/WorkflowResponse' + type: array + pagination: + $ref: '#/components/schemas/PaginationInfo' + required: + - data + - pagination + type: object + WorkflowPublishInfo: + description: Publishing metadata for a workflow shared to the Hub. + properties: + assets: + description: Published assets (inputs and models). + items: + $ref: '#/components/schemas/AssetInfo' + type: array + listed: + type: boolean + publish_time: + format: date-time + nullable: true + type: string + share_id: + type: string + workflow_id: + type: string + required: + - workflow_id + - share_id + - listed + - assets + type: object + WorkflowResponse: + description: Full workflow entity including metadata and version history. + properties: + created_at: + format: date-time + type: string + created_by: + type: string + default_view: + enum: + - workflow + - app + type: string + description: + type: string + forked_from: + $ref: '#/components/schemas/WorkflowForkedFrom' + id: + type: string + latest_version: + type: integer + name: + type: string + updated_at: + format: date-time + type: string + required: + - id + - latest_version + - created_by + - created_at + - updated_at + type: object + WorkflowVersionContentResponse: + description: Full workflow version including the serialized workflow JSON. + properties: + created_at: + format: date-time + type: string + created_by: + type: string + dependency_asset_ids: + items: + type: string + type: array + id: + type: string + version: + type: integer + workflow_json: + additionalProperties: true + type: object + required: + - id + - version + - workflow_json + - created_by + - created_at + type: object + WorkflowVersionResponse: + description: Metadata for a single workflow version. + properties: + created_at: + format: date-time + type: string + created_by: + type: string + id: + type: string + latest_version: + type: integer + version: + type: integer + required: + - id + - version + - latest_version + - created_by + - created_at + type: object + securitySchemes: + ApiKeyAuth: + description: | + API key authentication. Keys are prefixed with 'comfyui-' and can be + generated from user account settings. Example: 'comfyui-abc123...' + in: header + name: X-API-Key + type: apiKey + BearerAuth: + bearerFormat: JWT + description: | + Firebase JWT token authentication. Obtain a token by authenticating + with Firebase and pass it in the Authorization header. + scheme: bearer + type: http + CookieAuth: + description: | + Session cookie authentication. Set automatically after successful + login via the /api/auth/session endpoint. + in: cookie + name: session + type: apiKey +info: + description: | + API for ComfyUI - A powerful and modular UI for Stable Diffusion. - Member: - type: object - x-runtime: [cloud] - description: '[cloud-only] Workspace member with profile and role information.' - required: - - id - - name - - email - - role - - joined_at - properties: - id: - type: string - description: User ID - name: - type: string - description: User's display name - email: - type: string - format: email - description: User's email address - role: - type: string - enum: - - owner - - member - description: User's role in the workspace - joined_at: - type: string - format: date-time - description: When the user joined the workspace + This API allows you to interact with ComfyUI programmatically, including: + - Retrieving prompt information + - Retrieving node information + license: + name: GNU General Public License v3.0 + url: https://github.com/Comfy-Org/ComfyUI/blob/master/LICENSE + title: ComfyUI API + version: 1.0.0 +openapi: 3.0.3 +paths: + /api/assets: + get: + description: | + Retrieves a paginated list of assets belonging to the authenticated user. + Supports filtering by tags, name, metadata, and sorting options. + operationId: listAssets + parameters: + - description: Filter assets that have ALL of these tags + explode: false + in: query + name: include_tags + schema: + items: + type: string + type: array + style: form + - description: Exclude assets that have ANY of these tags + explode: false + in: query + name: exclude_tags + schema: + items: + type: string + type: array + style: form + - description: Filter assets where name contains this substring (case-insensitive) + in: query + name: name_contains + schema: + type: string + - description: JSON object for filtering by metadata fields + in: query + name: metadata_filter + schema: + type: string + - description: Maximum number of assets to return (1-500) + in: query + name: limit + schema: + default: 20 + maximum: 500 + minimum: 1 + type: integer + - description: Number of assets to skip for pagination + in: query + name: offset + schema: + default: 0 + minimum: 0 + type: integer + - description: Field to sort by + in: query + name: sort + schema: + default: created_at + enum: + - name + - created_at + - updated_at + - size + - last_access_time + type: string + - description: Sort order + in: query + name: order + schema: + default: desc + enum: + - asc + - desc + type: string + - description: Whether to include public/shared assets in results + in: query + name: include_public + schema: + default: true + type: boolean + - description: Filter assets by exact content hash. + in: query + name: hash + schema: + type: string + - description: | + Opaque cursor for keyset pagination. Pass the `next_cursor` value + from the previous response to fetch the next page. When provided, + `offset` is ignored. Cursor pagination is only supported with + `sort` values `created_at`, `updated_at`, `name`, or `size`; + requests combining `after` with other sort fields return 400. + The cursor must have been minted under the same `sort` value used + in the follow-up request. + in: query + name: after + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ListAssetsResponse' + description: Success - Assets returned + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Invalid request parameters + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Unauthorized + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Internal server error + summary: List user assets + tags: + - file + post: + description: | + Creates a new asset from a direct file upload (multipart/form-data) with associated metadata. - OAuthRegisterBadRequestResponse: - x-runtime: [cloud] - description: "[cloud-only] Union of the two 400 shapes /oauth/register can emit. `OAuthRegisterError` is the handler-shaped\ - \ RFC 7591 \xA73.2.2 error; `BindingErrorResponse` is the strict-server binding-layer error fired when the request body\ - \ fails OpenAPI-schema validation before the handler runs.\n" - oneOf: - - $ref: '#/components/schemas/OAuthRegisterError' - - $ref: '#/components/schemas/BindingErrorResponse' + If an asset with the same hash already exists, returns the existing asset. + operationId: createAsset + requestBody: + content: + multipart/form-data: + schema: + properties: + file: + description: The asset file to upload + format: binary + type: string + hash: + description: Content hash of the file. + pattern: ^(blake3|sha256):[a-f0-9]{64}$ + type: string + id: + description: Optional asset ID for idempotent creation. If provided and asset exists, returns existing asset. + format: uuid + type: string + mime_type: + description: MIME type of the asset (e.g., "image/png", "video/mp4") + type: string + name: + description: Display name for the asset + type: string + preview_id: + description: Optional preview asset ID. If not provided, images will use their own ID as preview. + format: uuid + type: string + tags: + description: JSON-encoded array of freeform tag strings, e.g. '["models","checkpoint"]'. Common types include "models", "input", "output", and "temp", but any tag can be used in any order. + type: string + user_metadata: + description: Custom JSON metadata as a string + type: string + required: + - file + type: object + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/AssetCreated' + description: | + Asset already existed for this user (deduplicated by content hash); the + existing asset is returned with created_new=false. + "201": + content: + application/json: + schema: + $ref: '#/components/schemas/AssetCreated' + description: Asset created successfully (created_new=true) + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Invalid request (bad file, invalid content type, etc.) + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Unauthorized + "413": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: File too large + "415": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Unsupported media type + "422": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Validation error (e.g., disallowed model_type tag) + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Internal server error + summary: Create a new asset + tags: + - file + /api/assets/{id}: + delete: + description: Deletes the asset record. + operationId: deleteAsset + parameters: + - description: Asset ID + in: path + name: id + required: true + schema: + format: uuid + type: string + responses: + "204": + description: Asset record deleted successfully + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Unauthorized + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Asset not found + "409": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 'Asset cannot be deleted because it is referenced by another resource, e.g. a workflow version (error code: ASSET_IN_USE)' + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Internal server error + summary: Delete asset + tags: + - file + get: + description: Retrieves detailed information about a specific asset + operationId: getAssetById + parameters: + - description: Asset ID + in: path + name: id + required: true + schema: + format: uuid + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/Asset' + description: Asset details retrieved successfully + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Unauthorized + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Asset not found + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Internal server error + summary: Get asset details + tags: + - file + put: + description: | + Updates an asset's metadata. At least one field must be provided. + Only name, mime_type, preview_id, and user_metadata can be updated. + For tag management, use POST (add) and DELETE (remove) /api/assets/{id}/tags. + operationId: updateAsset + parameters: + - description: Asset ID + in: path + name: id + required: true + schema: + format: uuid + type: string + requestBody: + content: + application/json: + schema: + minProperties: 1 + properties: + mime_type: + description: Updated MIME type of the asset + type: string + name: + description: New display name for the asset + type: string + preview_id: + description: Updated preview asset ID + format: uuid + type: string + user_metadata: + additionalProperties: true + description: Updated custom metadata + type: object + type: object + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/AssetUpdated' + description: Asset updated successfully + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: | + Invalid request — no fields provided, or `preview_id` is the zero UUID + (`INVALID_PREVIEW_ID`). + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Unauthorized + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: | + Asset not found — returned both when the asset being updated does + not exist and when `preview_id` does not reference an asset + accessible to the caller. + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Internal server error + summary: Update asset metadata + tags: + - file + /api/assets/{id}/content: + get: + description: | + Returns the binary content of an asset by ID. - PendingInvite: - type: object - x-runtime: [cloud] - description: '[cloud-only] An outstanding workspace invitation that has not yet been accepted.' - required: - - id - - email - - invited_at - - expires_at - properties: - id: - type: string - description: Invite ID - email: - type: string - format: email - description: Email address of the invited user - token: - type: string - description: Invite token for constructing invite links. Empty for expired invites. - invited_at: - type: string - format: date-time - description: When the invite was created - expires_at: - type: string - format: date-time - description: When the invite expires + The contract is the same across runtimes — "GET this path and you + receive the asset's bytes" — but the mechanism differs: + - **Local ComfyUI** streams the bytes directly (`200`, + `application/octet-stream`). + - **Cloud** does not proxy large files; it responds `302` with a + `Location` redirect to a short-lived signed storage URL. Clients that + follow redirects (browsers, `fetch`/XHR, ``/`