diff --git a/.gitignore b/.gitignore index e29ac43..d2a4293 100644 --- a/.gitignore +++ b/.gitignore @@ -1,179 +1,179 @@ -hf_download/ -outputs/ -repo/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -.idea/ - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc -demo_gradio.py \ No newline at end of file +# hf_download/ +# outputs/ +# repo/ + +# # Byte-compiled / optimized / DLL files +# __pycache__/ +# *.py[cod] +# *$py.class + +# # C extensions +# *.so + +# # Distribution / packaging +# .Python +# build/ +# develop-eggs/ +# dist/ +# downloads/ +# eggs/ +# .eggs/ +# lib/ +# lib64/ +# parts/ +# sdist/ +# var/ +# wheels/ +# share/python-wheels/ +# *.egg-info/ +# .installed.cfg +# *.egg +# MANIFEST + +# # PyInstaller +# # Usually these files are written by a python script from a template +# # before PyInstaller builds the exe, so as to inject date/other infos into it. +# *.manifest +# *.spec + +# # Installer logs +# pip-log.txt +# pip-delete-this-directory.txt + +# # Unit test / coverage reports +# htmlcov/ +# .tox/ +# .nox/ +# .coverage +# .coverage.* +# .cache +# nosetests.xml +# coverage.xml +# *.cover +# *.py,cover +# .hypothesis/ +# .pytest_cache/ +# cover/ + +# # Translations +# *.mo +# *.pot + +# # Django stuff: +# *.log +# local_settings.py +# db.sqlite3 +# db.sqlite3-journal + +# # Flask stuff: +# instance/ +# .webassets-cache + +# # Scrapy stuff: +# .scrapy + +# # Sphinx documentation +# docs/_build/ + +# # PyBuilder +# .pybuilder/ +# target/ + +# # Jupyter Notebook +# .ipynb_checkpoints + +# # IPython +# profile_default/ +# ipython_config.py + +# # pyenv +# # For a library or package, you might want to ignore these files since the code is +# # intended to run in multiple environments; otherwise, check them in: +# # .python-version + +# # pipenv +# # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# # However, in case of collaboration, if having platform-specific dependencies or dependencies +# # having no cross-platform support, pipenv may install dependencies that don't work, or not +# # install all needed dependencies. +# #Pipfile.lock + +# # UV +# # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# # This is especially recommended for binary packages to ensure reproducibility, and is more +# # commonly ignored for libraries. +# #uv.lock + +# # poetry +# # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# # This is especially recommended for binary packages to ensure reproducibility, and is more +# # commonly ignored for libraries. +# # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# #poetry.lock + +# # pdm +# # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# #pdm.lock +# # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# # in version control. +# # https://pdm.fming.dev/latest/usage/project/#working-with-version-control +# .pdm.toml +# .pdm-python +# .pdm-build/ + +# # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +# __pypackages__/ + +# # Celery stuff +# celerybeat-schedule +# celerybeat.pid + +# # SageMath parsed files +# *.sage.py + +# # Environments +# .env +# .venv +# env/ +# venv/ +# ENV/ +# env.bak/ +# venv.bak/ + +# # Spyder project settings +# .spyderproject +# .spyproject + +# # Rope project settings +# .ropeproject + +# # mkdocs documentation +# /site + +# # mypy +# .mypy_cache/ +# .dmypy.json +# dmypy.json + +# # Pyre type checker +# .pyre/ + +# # pytype static type analyzer +# .pytype/ + +# # Cython debug symbols +# cython_debug/ + +# # PyCharm +# # JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# # and can be added to the global gitignore or merged into this file. For a more nuclear +# # option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# # Ruff stuff: +# .ruff_cache/ + +# # PyPI configuration file +# .pypirc +# demo_gradio.py \ No newline at end of file diff --git a/README.md b/README.md index c739ce3..3d49ae4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # ComfyUI Wrapper for [FramePack by lllyasviel](https://lllyasviel.github.io/frame_pack_gitpage/) +# What have I modified? + +![微信截图_20250430184400](https://github.com/user-attachments/assets/94e7ed1d-109d-410f-927d-67092f21d918) + +I tried to add Hunyuan LoRA to it. Currently, most of the keys are matched correctly, but I'm not sure if it's really effective. I hope this is a start to attract more valuable opinions. +You can find the example workflow at \example_workflows\framepack-with-lora_hv_example.json + # WORK IN PROGRESS Mostly working, took some liberties to make it run faster. diff --git a/diffusers_helper/__pycache__/bucket_tools.cpython-311.pyc b/diffusers_helper/__pycache__/bucket_tools.cpython-311.pyc new file mode 100644 index 0000000..e8ca851 Binary files /dev/null and b/diffusers_helper/__pycache__/bucket_tools.cpython-311.pyc differ diff --git a/diffusers_helper/__pycache__/dit_common.cpython-311.pyc b/diffusers_helper/__pycache__/dit_common.cpython-311.pyc new file mode 100644 index 0000000..26b87e3 Binary files /dev/null and b/diffusers_helper/__pycache__/dit_common.cpython-311.pyc differ diff --git a/diffusers_helper/__pycache__/memory.cpython-311.pyc b/diffusers_helper/__pycache__/memory.cpython-311.pyc new file mode 100644 index 0000000..603a314 Binary files /dev/null and b/diffusers_helper/__pycache__/memory.cpython-311.pyc differ diff --git a/diffusers_helper/__pycache__/utils.cpython-311.pyc b/diffusers_helper/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000..dd208fb Binary files /dev/null and b/diffusers_helper/__pycache__/utils.cpython-311.pyc differ diff --git a/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-311.pyc b/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-311.pyc new file mode 100644 index 0000000..6f7317e Binary files /dev/null and b/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-311.pyc differ diff --git a/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-311.pyc b/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-311.pyc new file mode 100644 index 0000000..62efd01 Binary files /dev/null and b/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-311.pyc differ diff --git a/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-311.pyc b/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-311.pyc new file mode 100644 index 0000000..3f9a75e Binary files /dev/null and b/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-311.pyc differ diff --git a/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-311.pyc b/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-311.pyc new file mode 100644 index 0000000..f7b80b2 Binary files /dev/null and b/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-311.pyc differ diff --git a/example_workflows/framepack-with-lora_hv_example.json b/example_workflows/framepack-with-lora_hv_example.json new file mode 100644 index 0000000..1046616 --- /dev/null +++ b/example_workflows/framepack-with-lora_hv_example.json @@ -0,0 +1,2378 @@ +{ + "id": "ce2cb810-7775-4564-8928-dd5bed1053cd", + "revision": 0, + "last_node_id": 93, + "last_link_id": 167, + "nodes": [ + { + "id": 15, + "type": "ConditioningZeroOut", + "pos": [ + 1346.0872802734375, + 263.21856689453125 + ], + "size": [ + 317.4000244140625, + 26 + ], + "flags": { + "collapsed": true + }, + "order": 27, + "mode": 0, + "inputs": [ + { + "label": "条件", + "name": "conditioning", + "type": "CONDITIONING", + "link": 118 + } + ], + "outputs": [ + { + "label": "条件", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 108 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "ConditioningZeroOut" + }, + "widgets_values": [], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 64, + "type": "GetNode", + "pos": [ + 1554.2071533203125, + 486.79547119140625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 149 + ] + } + ], + "title": "Get_ClipVisionModle", + "properties": {}, + "widgets_values": [ + "ClipVisionModle" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 60, + "type": "GetImageSizeAndCount", + "pos": [ + 1279.781494140625, + 1060.245361328125 + ], + "size": [ + 277.20001220703125, + 86 + ], + "flags": {}, + "order": 31, + "mode": 4, + "inputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 139 + } + ], + "outputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "links": [ + 151, + 152 + ] + }, + { + "label": "宽度", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "高度", + "name": "height", + "type": "INT", + "links": null + }, + { + "label": "数量", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", + "Node name for S&R": "GetImageSizeAndCount", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 20, + "type": "VAEEncode", + "pos": [ + 1733.111083984375, + 633.30419921875 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "pixels", + "type": "IMAGE", + "link": 156 + }, + { + "label": "VAE", + "name": "vae", + "type": "VAE", + "link": 155 + } + ], + "outputs": [ + { + "label": "Latent", + "name": "LATENT", + "type": "LATENT", + "links": [ + 86 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 68, + "type": "GetNode", + "pos": [ + 1729.60693359375, + 734.5352172851562 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 155 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 69, + "type": "GetNode", + "pos": [ + 1619.6104736328125, + 1137.854736328125 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 2, + "mode": 4, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 158 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 59, + "type": "ImageResize+", + "pos": [ + 908.9832763671875, + 1062.01123046875 + ], + "size": [ + 315, + 218 + ], + "flags": {}, + "order": 29, + "mode": 4, + "inputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 138 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 136 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 137 + } + ], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 139 + ] + }, + { + "label": "宽度", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "高度", + "name": "height", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui_essentials", + "ver": "76e9d1e4399bd025ce8b12c290753d58f9f53e93", + "Node name for S&R": "ImageResize+", + "aux_id": "kijai/ComfyUI_essentials", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + 512, + 512, + "lanczos", + "stretch", + "always", + 0 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 50, + "type": "ImageResize+", + "pos": [ + 907.2653198242188, + 593.743896484375 + ], + "size": [ + 315, + 218 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 122 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 128 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 127 + } + ], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 125 + ] + }, + { + "label": "宽度", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "高度", + "name": "height", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui_essentials", + "ver": "76e9d1e4399bd025ce8b12c290753d58f9f53e93", + "Node name for S&R": "ImageResize+", + "aux_id": "kijai/ComfyUI_essentials", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + 512, + 512, + "lanczos", + "stretch", + "always", + 0 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 18, + "type": "CLIPVisionLoader", + "pos": [ + 33.149566650390625, + 23.595293045043945 + ], + "size": [ + 388.87139892578125, + 58 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "CLIP视觉", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 148 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "sigclip_vision_patch14_384.safetensors" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 63, + "type": "SetNode", + "pos": [ + 247.1346435546875, + -28.502397537231445 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "link": 148 + } + ], + "outputs": [ + { + "label": "输出", + "name": "*", + "type": "*", + "links": null + } + ], + "properties": { + "previousName": "ClipVisionModle" + }, + "widgets_values": [ + "ClipVisionModle" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 27, + "type": "FramePackTorchCompileSettings", + "pos": [ + 623.3660278320312, + -140.94215393066406 + ], + "size": [ + 531.5999755859375, + 202 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "torch_compile_args", + "name": "torch_compile_args", + "type": "FRAMEPACKCOMPILEARGS", + "links": [] + } + ], + "properties": { + "aux_id": "lllyasviel/FramePack", + "ver": "0e5fe5d7ca13c76fb8e13708f4b92e7c7a34f20c", + "Node name for S&R": "FramePackTorchCompileSettings", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + true + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 33, + "type": "VAEDecodeTiled", + "pos": [ + 2328.923828125, + -22.08228874206543 + ], + "size": [ + 315, + 150 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "label": "Latent", + "name": "samples", + "type": "LATENT", + "link": 85 + }, + { + "label": "VAE", + "name": "vae", + "type": "VAE", + "link": 154 + } + ], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 96 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "VAEDecodeTiled" + }, + "widgets_values": [ + 256, + 64, + 64, + 8 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 67, + "type": "GetNode", + "pos": [ + 2342.01806640625, + -76.06847381591797 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 154 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 23, + "type": "VHS_VideoCombine", + "pos": [ + 2726.849853515625, + -29.90264129638672 + ], + "size": [ + 908.428955078125, + 1497.7315673828125 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "type": "IMAGE", + "link": 97 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "FramePack", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "FramePack_00004.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 30, + "workflow": "FramePack_00004.png", + "fullpath": "C:\\COMFYUI\\ComfyUI_windows_portable\\ComfyUI\\temp\\FramePack_00004.mp4" + } + } + }, + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 44, + "type": "GetImageSizeAndCount", + "pos": [ + 2501.023193359375, + -178.70773315429688 + ], + "size": [ + 277.20001220703125, + 86 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 96 + } + ], + "outputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "links": [ + 97 + ] + }, + { + "label": "544 width", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "704 height", + "name": "height", + "type": "INT", + "links": null + }, + { + "label": "145 count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", + "Node name for S&R": "GetImageSizeAndCount", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 51, + "type": "FramePackFindNearestBucket", + "pos": [ + 550.0997314453125, + 887.411376953125 + ], + "size": [ + 315, + 78 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "label": "image", + "name": "image", + "type": "IMAGE", + "link": 126 + } + ], + "outputs": [ + { + "label": "width", + "name": "width", + "type": "INT", + "links": [ + 128, + 136 + ] + }, + { + "label": "height", + "name": "height", + "type": "INT", + "links": [ + 127, + 137 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "4f9030a9f4c0bd67d86adf3d3dc07e37118c40bd", + "Node name for S&R": "FramePackFindNearestBucket", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + 640 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 58, + "type": "LoadImage", + "pos": [ + 190.07057189941406, + 1060.399169921875 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 6, + "mode": 4, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 138 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "LoadImage", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "image (2).webp", + "image" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 54, + "type": "DownloadAndLoadFramePackModel", + "pos": [ + 1256.5235595703125, + -277.76226806640625 + ], + "size": [ + 315, + 150 + ], + "flags": {}, + "order": 7, + "mode": 4, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "FRAMEPACKCOMPILEARGS", + "link": null + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "FRAMEPACKLORA", + "link": null + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "FramePackMODEL", + "links": null + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "49fe507eca8246cc9d08a8093892f40c1180e88f", + "Node name for S&R": "DownloadAndLoadFramePackModel", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "lllyasviel/FramePackI2V_HY", + "bf16", + "disabled", + "sdpa" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 70, + "type": "VAELoader", + "pos": [ + 19.38068199157715, + -296.1534118652344 + ], + "size": [ + 469.0488586425781, + 58 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "hyvid\\hunyuan_video_vae_bf16_repack.safetensors" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 71, + "type": "DualCLIPLoader", + "pos": [ + -168.35980224609375, + 263.41839599609375 + ], + "size": [ + 340.2243957519531, + 130 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "DualCLIPLoader" + }, + "widgets_values": [ + "clip_l.safetensors", + "llava_llama3_fp16.safetensors", + "hunyuan_video", + "default" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 72, + "type": "LoadFramePackModel", + "pos": [ + 2144.548583984375, + -453.7969055175781 + ], + "size": [ + 480.7601013183594, + 150 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "FRAMEPACKCOMPILEARGS", + "link": null + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "FRAMEPACKLORA", + "link": null + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "FramePackMODEL", + "links": [] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "49fe507eca8246cc9d08a8093892f40c1180e88f", + "Node name for S&R": "LoadFramePackModel", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "Hyvid\\FramePackI2V_HY_fp8_e4m3fn.safetensors", + "bf16", + "fp8_e4m3fn", + "sdpa" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 55, + "type": "MarkdownNote", + "pos": [ + 1627.5853271484375, + -415.2608642578125 + ], + "size": [ + 459.8609619140625, + 285.9714660644531 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Model links:\n\n[https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors](https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors)\n\n[https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors](https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors)\n\nsigclip:\n\n[https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main](https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main)\n\ntext encoder and VAE:\n\n[https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files](https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 13, + "type": "DualCLIPLoader", + "pos": [ + 262.6387634277344, + 146.843994140625 + ], + "size": [ + 340.2243957519531, + 130 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 102 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "DualCLIPLoader" + }, + "widgets_values": [ + "clip_l.safetensors", + "llava_llama3_fp16.safetensors", + "hunyuan_video", + "default" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 12, + "type": "VAELoader", + "pos": [ + 597.14453125, + -313.11004638671875 + ], + "size": [ + 469.0488586425781, + 58 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 153 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "hunyuan\\hunyuan_video_vae_bf16.safetensors" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 66, + "type": "SetNode", + "pos": [ + 1111.0233154296875, + -299.81610107421875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "VAE", + "type": "VAE", + "link": 153 + } + ], + "outputs": [ + { + "label": "输出", + "name": "*", + "type": "*", + "links": null + } + ], + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 75, + "type": "LoraLoaderModelOnly", + "pos": [ + 1210.9024658203125, + -470.9698181152344 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "模型", + "name": "model", + "type": "MODEL", + "link": null + } + ], + "outputs": [ + { + "label": "模型", + "name": "MODEL", + "type": "MODEL", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.29", + "Node name for S&R": "LoraLoaderModelOnly", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "1stPOV_flx_LoRA_v01.safetensors", + 1 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 17, + "type": "CLIPVisionEncode", + "pos": [ + 1545.9541015625, + 359.1331481933594 + ], + "size": [ + 380.4000244140625, + 78 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "label": "CLIP视觉", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 149 + }, + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 116 + } + ], + "outputs": [ + { + "label": "CLIP视觉输出", + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "links": [ + 141 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "CLIPVisionEncode" + }, + "widgets_values": [ + "center" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 48, + "type": "GetImageSizeAndCount", + "pos": [ + 1259.2060546875, + 626.8657836914062 + ], + "size": [ + 277.20001220703125, + 86 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 125 + } + ], + "outputs": [ + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "links": [ + 116, + 156 + ] + }, + { + "label": "544 width", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "704 height", + "name": "height", + "type": "INT", + "links": null + }, + { + "label": "1 count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", + "Node name for S&R": "GetImageSizeAndCount", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 57, + "type": "CLIPVisionEncode", + "pos": [ + 1600.4202880859375, + 1181.36767578125 + ], + "size": [ + 380.4000244140625, + 78 + ], + "flags": {}, + "order": 34, + "mode": 4, + "inputs": [ + { + "label": "CLIP视觉", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 150 + }, + { + "label": "图像", + "name": "image", + "type": "IMAGE", + "link": 151 + } + ], + "outputs": [ + { + "label": "CLIP视觉输出", + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "links": [ + 132 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.29", + "Node name for S&R": "CLIPVisionEncode" + }, + "widgets_values": [ + "center" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 62, + "type": "VAEEncode", + "pos": [ + 1612.563232421875, + 1048.6236572265625 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 35, + "mode": 4, + "inputs": [ + { + "label": "图像", + "name": "pixels", + "type": "IMAGE", + "link": 152 + }, + { + "label": "VAE", + "name": "vae", + "type": "VAE", + "link": 158 + } + ], + "outputs": [ + { + "label": "Latent", + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 65, + "type": "GetNode", + "pos": [ + 1604.746337890625, + 1306.3175048828125 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 15, + "mode": 4, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 150 + ] + } + ], + "title": "Get_ClipVisionModle", + "properties": {}, + "widgets_values": [ + "ClipVisionModle" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 80, + "type": "HyVideoLoraSelect", + "pos": [ + 1796.221923828125, + 181.3782958984375 + ], + "size": [ + 315, + 102 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "label": "prev_lora", + "name": "prev_lora", + "shape": 7, + "type": "HYVIDLORA", + "link": null + }, + { + "label": "blocks", + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "label": "lora", + "name": "lora", + "type": "HYVIDLORA", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui-hunyuanvideowrapper", + "ver": "83f0bbb8694cf92e994189b045270aab4e5029b5", + "Node name for S&R": "HyVideoLoraSelect", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "1stPOV_flx_LoRA_v01.safetensors", + 1 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 81, + "type": "HyVideoModelLoader", + "pos": [ + 1742.34228515625, + -823.1158447265625 + ], + "size": [ + 426.1773986816406, + 242 + ], + "flags": {}, + "order": 23, + "mode": 4, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "COMPILEARGS", + "link": null + }, + { + "label": "block_swap_args", + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": null + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "HYVIDLORA", + "link": 163 + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "HYVIDEOMODEL", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui-hunyuanvideowrapper", + "ver": "83f0bbb8694cf92e994189b045270aab4e5029b5", + "Node name for S&R": "HyVideoModelLoader", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "FramePackI2V_HY_fp8_e4m3fn.safetensors", + "bf16", + "fp8_e4m3fn", + "offload_device", + "sageattn_varlen", + false, + true + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 82, + "type": "HyVideoLoraSelect", + "pos": [ + 1509.4305419921875, + -1198.0078125 + ], + "size": [ + 315, + 102 + ], + "flags": {}, + "order": 17, + "mode": 4, + "inputs": [ + { + "label": "prev_lora", + "name": "prev_lora", + "shape": 7, + "type": "HYVIDLORA", + "link": null + }, + { + "label": "blocks", + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "label": "lora", + "name": "lora", + "type": "HYVIDLORA", + "links": [ + 163 + ] + } + ], + "properties": { + "cnr_id": "comfyui-hunyuanvideowrapper", + "ver": "83f0bbb8694cf92e994189b045270aab4e5029b5", + "Node name for S&R": "HyVideoLoraSelect", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "hyvid_I2V_lora_embrace.safetensors", + 1 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 92, + "type": "LoadFramePackModelWithLoRA", + "pos": [ + 2616.2822265625, + -713.7622680664062 + ], + "size": [ + 451.09912109375, + 150.69114685058594 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "FRAMEPACKCOMPILEARGS", + "link": null + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "FRAMEPACKLORA", + "link": 166 + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "FramePackMODEL", + "links": [ + 167 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "dcb0fc64d6266be728457a62c8737e1192605f20", + "Node name for S&R": "LoadFramePackModelWithLoRA", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "FramePackI2V_HY_fp8_e4m3fn.safetensors", + "bf16", + "fp8_e4m3fn", + "sdpa" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 87, + "type": "FramePackLoraSelect", + "pos": [ + 2218.852294921875, + -689.4024658203125 + ], + "size": [ + 315, + 102 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "label": "prev_lora", + "name": "prev_lora", + "shape": 7, + "type": "FRAMEPACKLORA", + "link": null + }, + { + "label": "blocks", + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "label": "lora", + "name": "lora", + "type": "FRAMEPACKLORA", + "links": [ + 165, + 166 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "dcb0fc64d6266be728457a62c8737e1192605f20", + "Node name for S&R": "FramePackLoraSelect", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "tittydrop_v1.1.safetensors", + 1 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 83, + "type": "LoadFramePackModel", + "pos": [ + 2653.668212890625, + -1214.444580078125 + ], + "size": [ + 391.71453857421875, + 161.0579071044922 + ], + "flags": {}, + "order": 24, + "mode": 4, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "FRAMEPACKCOMPILEARGS", + "link": null + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "FRAMEPACKLORA", + "link": 165 + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "FramePackMODEL", + "links": [] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "dcb0fc64d6266be728457a62c8737e1192605f20", + "Node name for S&R": "LoadFramePackModel", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "FramePackI2V_HY_fp8_e4m3fn.safetensors", + "bf16", + "fp8_e4m3fn", + "sdpa" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 19, + "type": "LoadImage", + "pos": [ + 184.2612762451172, + 591.6886596679688 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 122, + 126 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "LoadImage", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + "ComfyUI_temp_llhmr_00024_@1x.png", + "image" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 47, + "type": "CLIPTextEncode", + "pos": [ + 711.8784790039062, + 146.58335876464844 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "label": "CLIP", + "name": "clip", + "type": "CLIP", + "link": 102 + } + ], + "outputs": [ + { + "label": "条件", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 114, + 118 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.28", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "t1ttydr0p, A professional video of a girl. She lifts up her top shirt. After a second, she reveals her breasts and nipples. navel. Detailed smiling face with gorgeous makeup, high resolution." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 39, + "type": "FramePackSampler", + "pos": [ + 2292.58837890625, + 194.90232849121094 + ], + "size": [ + 365.07305908203125, + 814.6473388671875 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "label": "model", + "name": "model", + "type": "FramePackMODEL", + "link": 167 + }, + { + "label": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 114 + }, + { + "label": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 108 + }, + { + "label": "image_embeds", + "name": "image_embeds", + "type": "CLIP_VISION_OUTPUT", + "link": 141 + }, + { + "label": "start_latent", + "name": "start_latent", + "shape": 7, + "type": "LATENT", + "link": 86 + }, + { + "label": "end_latent", + "name": "end_latent", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "label": "end_image_embeds", + "name": "end_image_embeds", + "shape": 7, + "type": "CLIP_VISION_OUTPUT", + "link": 132 + }, + { + "label": "initial_samples", + "name": "initial_samples", + "shape": 7, + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "label": "samples", + "name": "samples", + "type": "LATENT", + "links": [ + 85 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-FramePackWrapper", + "ver": "8e5ec6b7f3acf88255c5d93d062079f18b43aa2b", + "Node name for S&R": "FramePackSampler", + "ttNbgOverride": { + "color": "#332922", + "bgcolor": "#593930", + "groupcolor": "#b06634" + } + }, + "widgets_values": [ + 30, + true, + 0.15, + 1, + 10, + 0, + 887596242712916, + "randomize", + 9, + 5, + 6, + "unipc_bh1", + "weighted_average", + 0.5, + 1 + ], + "color": "#332922", + "bgcolor": "#593930" + } + ], + "links": [ + [ + 85, + 39, + 0, + 33, + 0, + "LATENT" + ], + [ + 86, + 20, + 0, + 39, + 4, + "LATENT" + ], + [ + 96, + 33, + 0, + 44, + 0, + "IMAGE" + ], + [ + 97, + 44, + 0, + 23, + 0, + "IMAGE" + ], + [ + 102, + 13, + 0, + 47, + 0, + "CLIP" + ], + [ + 108, + 15, + 0, + 39, + 2, + "CONDITIONING" + ], + [ + 114, + 47, + 0, + 39, + 1, + "CONDITIONING" + ], + [ + 116, + 48, + 0, + 17, + 1, + "IMAGE" + ], + [ + 118, + 47, + 0, + 15, + 0, + "CONDITIONING" + ], + [ + 122, + 19, + 0, + 50, + 0, + "IMAGE" + ], + [ + 125, + 50, + 0, + 48, + 0, + "IMAGE" + ], + [ + 126, + 19, + 0, + 51, + 0, + "IMAGE" + ], + [ + 127, + 51, + 1, + 50, + 2, + "INT" + ], + [ + 128, + 51, + 0, + 50, + 1, + "INT" + ], + [ + 132, + 57, + 0, + 39, + 6, + "CLIP_VISION_OUTPUT" + ], + [ + 136, + 51, + 0, + 59, + 1, + "INT" + ], + [ + 137, + 51, + 1, + 59, + 2, + "INT" + ], + [ + 138, + 58, + 0, + 59, + 0, + "IMAGE" + ], + [ + 139, + 59, + 0, + 60, + 0, + "IMAGE" + ], + [ + 141, + 17, + 0, + 39, + 3, + "CLIP_VISION_OUTPUT" + ], + [ + 148, + 18, + 0, + 63, + 0, + "*" + ], + [ + 149, + 64, + 0, + 17, + 0, + "CLIP_VISION" + ], + [ + 150, + 65, + 0, + 57, + 0, + "CLIP_VISION" + ], + [ + 151, + 60, + 0, + 57, + 1, + "IMAGE" + ], + [ + 152, + 60, + 0, + 62, + 0, + "IMAGE" + ], + [ + 153, + 12, + 0, + 66, + 0, + "*" + ], + [ + 154, + 67, + 0, + 33, + 1, + "VAE" + ], + [ + 155, + 68, + 0, + 20, + 1, + "VAE" + ], + [ + 156, + 48, + 0, + 20, + 0, + "IMAGE" + ], + [ + 158, + 69, + 0, + 62, + 1, + "VAE" + ], + [ + 163, + 82, + 0, + 81, + 2, + "HYVIDLORA" + ], + [ + 165, + 87, + 0, + 83, + 1, + "FRAMEPACKLORA" + ], + [ + 166, + 87, + 0, + 92, + 1, + "FRAMEPACKLORA" + ], + [ + 167, + 92, + 0, + 39, + 0, + "FramePackMODEL" + ] + ], + "groups": [ + { + "id": 1, + "title": "End Image", + "bounding": [ + 12.77297592163086, + 999.1203002929688, + 2038.674560546875, + 412.9618225097656 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Start Image", + "bounding": [ + 11.781991958618164, + 531.3884887695312, + 2032.7288818359375, + 442.6904602050781 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.40909090909091134, + "offset": [ + 5.876397287110876, + 864.1282410963076 + ] + }, + "frontendVersion": "1.16.8", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "ue_links": [] + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index 3e72401..c68c1c2 100644 --- a/nodes.py +++ b/nodes.py @@ -16,6 +16,438 @@ import comfy.latent_formats from comfy.cli_args import args, LatentPreviewMethod +def filter_state_dict_by_blocks(state_dict, blocks_mapping): + filtered_dict = {} + + for key in state_dict: + if 'transformer_blocks.' in key or 'single_transformer_blocks.' in key: + block_pattern = key.split('transformer.')[1].split('.', 2)[0:2] + block_key = f'{block_pattern[0]}.{block_pattern[1]}.' + + if block_key in blocks_mapping: + filtered_dict[key] = state_dict[key] + + return filtered_dict + +def standardize_lora_key_format(lora_sd): + new_sd = {} + for k, v in lora_sd.items(): + # Diffusers format + if k.startswith('transformer.'): + k = k.replace('transformer.', '') + if "img_attn.proj" in k: + k = k.replace("img_attn.proj", "img_attn_proj") + if "img_attn.qkv" in k: + k = k.replace("img_attn.qkv", "img_attn_qkv") + if "txt_attn.proj" in k: + k = k.replace("txt_attn.proj ", "txt_attn_proj") + if "txt_attn.qkv" in k: + k = k.replace("txt_attn.qkv", "txt_attn_qkv") + new_sd[k] = v + return new_sd + +def apply_lora_weights_manually(transformer, lora_sd, lora_strength): + # 手动应用LoRA权重，不依赖ComfyUI的load_lora_for_models + # transformer: 原始模型 + # lora_sd: LoRA权重state_dict + # lora_strength: LoRA强度 + + matched_count = 0 # 匹配成功的参数计数 + total_pairs = 0 # 找到的LoRA对总数 + + # 输出LoRA文件中前10个键，方便分析格式 + print(f"LoRA文件包含 {len(lora_sd)} 个权重键") + key_samples = list(lora_sd.keys())[:10] + print(f"LoRA键名示例: {', '.join(key_samples)}") + + # 检测不同的LoRA格式 + lora_format = "unknown" + if any(".lora_up.weight" in k for k in lora_sd.keys()): + lora_format = "standard" # 标准格式：key.lora_up.weight/key.lora_down.weight + elif any(".weight" in k and ".alpha" in k for k in lora_sd.keys()): + lora_format = "kohya" # Kohya格式：key.weight/key.alpha + elif any("_lora_a.weight" in k for k in lora_sd.keys()): + lora_format = "diffusers" # Diffusers格式：key_lora_a.weight/key_lora_b.weight + elif any(".lora_A.weight" in k for k in lora_sd.keys()): + lora_format = "lycoris" # LyCORIS格式：key.lora_A.weight/key.lora_B.weight + + print(f"检测到LoRA格式: {lora_format}") + + # 用于记录键对的字典 + up_down_pairs = {} + + # 根据不同格式找出所有up-down对 + if lora_format == "standard": + # 标准格式: key.lora_up.weight/key.lora_down.weight + for lora_key in lora_sd.keys(): + if ".lora_up.weight" in lora_key: + base_key = lora_key.replace(".lora_up.weight", "") + down_key = base_key + ".lora_down.weight" + + if down_key in lora_sd: + # 提取目标参数名 - 尝试多种格式 + # 1. 完整路径最后一部分 + target_key_simple = base_key.split(".")[-1] + # 2. 最后两部分 (如 self_attn.v_proj) + target_key_compound = '.'.join(base_key.split(".")[-2:]) if len(base_key.split(".")) >= 2 else target_key_simple + + up_down_pairs[target_key_simple] = (lora_key, down_key) + if target_key_simple != target_key_compound: + up_down_pairs[target_key_compound] = (lora_key, down_key) + total_pairs += 1 + + elif lora_format == "kohya": + # Kohya格式: key.weight/key.alpha + weights = {} + alphas = {} + + for k in lora_sd.keys(): + if ".weight" in k and not ".alpha" in k: + weights[k.replace(".weight", "")] = k + elif ".alpha" in k: + alphas[k.replace(".alpha", "")] = k + + for base in weights.keys(): + if base in alphas: + up_key = weights[base] + alpha_key = alphas[base] + # 在Kohya格式中，同一个key包含了up和down权重 + target_key_simple = base.split(".")[-1] + target_key_compound = '.'.join(base.split(".")[-2:]) if len(base.split(".")) >= 2 else target_key_simple + + # 存储(up键, down键)tuple，在Kohya格式中下两个值相同 + up_down_pairs[target_key_simple] = (up_key, up_key) + if target_key_simple != target_key_compound: + up_down_pairs[target_key_compound] = (up_key, up_key) + total_pairs += 1 + + elif lora_format == "diffusers": + # Diffusers格式: key_lora_a.weight/key_lora_b.weight + for lora_key in lora_sd.keys(): + if "_lora_a.weight" in lora_key: + base_key = lora_key.replace("_lora_a.weight", "") + down_key = base_key + "_lora_b.weight" + + if down_key in lora_sd: + target_key_simple = base_key.split(".")[-1] + target_key_compound = '.'.join(base_key.split(".")[-2:]) if len(base_key.split(".")) >= 2 else target_key_simple + + up_down_pairs[target_key_simple] = (lora_key, down_key) + if target_key_simple != target_key_compound: + up_down_pairs[target_key_compound] = (lora_key, down_key) + total_pairs += 1 + elif lora_format == "lycoris": + # LyCORIS格式: key.lora_A.weight/key.lora_B.weight + for lora_key in lora_sd.keys(): + if ".lora_A.weight" in lora_key: + base_key = lora_key.replace(".lora_A.weight", "") + down_key = base_key + ".lora_B.weight" + + if down_key in lora_sd: + # 尝试多种提取目标键的方式 + parts = base_key.split(".") + + # 尝试提取目标模块名 + target_keys = [] + # 1. 只取最后一部分（例如img_mlp.fc1的fc1） + target_keys.append(parts[-1]) + # 2. 取最后两部分（例如img_mlp.fc1） + if len(parts) >= 2: + target_keys.append(f"{parts[-2]}.{parts[-1]}") + # 3. 也可能是分开的两部分 + target_keys.append(parts[-2]) + # 4. 如果是double_blocks或single_transformer_blocks格式，取出相关部分 + if "double_blocks" in base_key or "transformer_blocks" in base_key or "single_transformer_blocks" in base_key: + for i in range(len(parts)): + if parts[i] in ["double_blocks", "transformer_blocks", "single_transformer_blocks"] and i+2 < len(parts): + # 尝试用block_idx.module_name格式 + target_keys.append(f"{parts[i+1]}.{parts[i+2]}") + # 单独的模块名 + target_keys.append(parts[i+2]) + + # 将所有可能的目标键添加到字典 + for key in set(target_keys): # 使用set去除重复 + up_down_pairs[key] = (lora_key, down_key) + + total_pairs += 1 + else: + # 尝试直接匹配模型的权重键 + print("未知LoRA格式，尝试直接匹配...") + for key in lora_sd.keys(): + if ".weight" in key: + base_key = key.replace(".weight", "") + # 提取多种可能的目标键 + parts = base_key.split(".") + if len(parts) > 0: + target_key_simple = parts[-1] + up_down_pairs[target_key_simple] = (key, key) # 同一个键作为up和down + # 如果有两部分以上，尝试当做复合名称 + if len(parts) >= 2: + target_key_compound = f"{parts[-2]}.{parts[-1]}" + up_down_pairs[target_key_compound] = (key, key) + total_pairs += 1 + + print(f"找到 {len(up_down_pairs)} 个有效的LoRA up-down对") + + # 收集模型中的所有可能目标 + model_module_names = set() + model_module_paths = {} + + def collect_module_names(module, name_prefix=""): + for name, child in module.named_children(): + full_name = name_prefix + "." + name if name_prefix else name + + # 添加各种可能的模块名称形式 + model_module_names.add(name) # 简单名称 + model_module_paths[name] = full_name # 记录完整路径 + + # 复合名称 (例如: parent.child) + if len(name_prefix.split(".")) > 0: + parent = name_prefix.split(".")[-1] + compound_name = f"{parent}.{name}" + model_module_names.add(compound_name) + model_module_paths[compound_name] = full_name + + # 如果是transformer块，添加特殊匹配 + if ("double_blocks" in name_prefix or "transformer_blocks" in name_prefix or + "single_transformer_blocks" in name_prefix): + # 提取块索引 + parts = name_prefix.split(".") + for i, part in enumerate(parts): + if part in ["double_blocks", "transformer_blocks", "single_transformer_blocks"] and i+1 < len(parts): + block_idx = parts[i+1] + # 块索引.module格式 + block_module = f"{block_idx}.{name}" + model_module_names.add(block_module) + model_module_paths[block_module] = full_name + + # 递归处理子模块 + collect_module_names(child, full_name) + + collect_module_names(transformer) + print(f"模型中有 {len(model_module_names)} 个可能的模块名称") + print(f"模型模块示例: {', '.join(sorted(list(model_module_names))[:5])}{'...' if len(model_module_names) > 5 else ''}") + + # 查找LoRA键与模型模块名称的交集 + common_keys = set(up_down_pairs.keys()).intersection(model_module_names) + print(f"找到 {len(common_keys)} 个可能匹配的模块名称") + if common_keys: + print(f"匹配的模块名称: {', '.join(sorted(list(common_keys))[:5])}{'...' if len(common_keys) > 5 else ''}") + + # 打印LoRA中的一些键来帮助诊断 + print(f"未匹配的LoRA键的示例: {', '.join(list(up_down_pairs.keys())[:5])}") + + # 如果没有匹配，尝试更宽松的匹配方式 + if len(common_keys) == 0 and lora_format == "lycoris": + print("没有找到精确匹配，尝试部分字符串匹配...") + # 创建一个映射来存储模糊匹配 + fuzzy_matches = {} + + for lora_key in up_down_pairs.keys(): + best_match = None + best_score = 0 + + for model_key in model_module_names: + # 基本匹配: 如果一个是另一个的子字符串 + if lora_key in model_key or model_key in lora_key: + score = len(set(lora_key).intersection(set(model_key))) + if score > best_score: + best_score = score + best_match = model_key + + if best_match and best_score > len(lora_key) / 2: # 至少半数字符匹配 + fuzzy_matches[lora_key] = best_match + + if fuzzy_matches: + print(f"模糊匹配到 {len(fuzzy_matches)} 个可能的模块") + for lora_key, model_key in list(fuzzy_matches.items())[:5]: + print(f" LoRA键: '{lora_key}' 匹配到模型模块: '{model_key}'") + + # 将模糊匹配添加到common_keys + for lora_key, model_key in fuzzy_matches.items(): + common_keys.add(model_key) + + # 递归遍历模型的所有参数 + applied_modules = set() # 记录已经应用过LoRA的模块路径 + + # 查找直接匹配的模块路径，获取直接的参数引用 + for model_key in common_keys: + if model_key in model_module_paths: + full_path = model_module_paths[model_key] + parts = full_path.split('.') + + # 试图获取指定路径的模块 + current = transformer + for part in parts: + if hasattr(current, part): + current = getattr(current, part) + else: + current = None + break + + if current is not None and hasattr(current, 'weight'): + # 直接获得了模块，并且有weight参数 + param = current.weight + up_key, down_key = up_down_pairs[model_key] + up_weight = lora_sd[up_key] + down_weight = lora_sd[down_key] + + # 当使用LyCORIS格式时，需要尝试不同的矩阵计算 + try: + if lora_format == "lycoris": + # LyCORIS格式通常的A/B矩阵需要特殊处理 + print(f"LyCORIS格式: 尝试应用到 {full_path}.weight") + print(f" 参数形状: {param.shape}, A形状: {up_weight.shape}, B形状: {down_weight.shape}") + + # 由于LoRA格式变化，尝试多种矩阵乘法 + delta = None + + # 1. 尝试标准LoRA方式：B x A + if down_weight.shape[1] == up_weight.shape[0]: + delta = torch.mm(down_weight, up_weight) * lora_strength + print(" 使用标准LoRA矩阵乘法: B x A") + + # 2. 尝试反转方式：A x B + elif up_weight.shape[1] == down_weight.shape[0]: + delta = torch.mm(up_weight, down_weight) * lora_strength + print(" 使用反转矩阵乘法: A x B") + + # 3. 尝试转置矩阵 + elif up_weight.shape[0] == down_weight.shape[0]: + delta = torch.mm(up_weight.t(), down_weight) * lora_strength + print(" 使用转置矩阵乘法: A^T x B") + elif up_weight.shape[1] == down_weight.shape[1]: + delta = torch.mm(up_weight, down_weight.t()) * lora_strength + print(" 使用转置矩阵乘法: A x B^T") + + if delta is not None: + print(f" 计算得到delta形状: {delta.shape}") + # 如果形状直接匹配 + if delta.shape == param.shape: + param.data += delta + matched_count += 1 + applied_modules.add(full_path) + print(f" 直接应用LoRA到 {full_path}.weight") + # 尝试reshape + else: + try: + delta_reshaped = delta.reshape(param.shape) + param.data += delta_reshaped + matched_count += 1 + applied_modules.add(full_path) + print(f" 通过reshape成功应用LoRA到 {full_path}.weight") + except Exception as e: + print(f" 无法reshape到参数形状: {str(e)}") + else: + # 其他LoRA格式的标准处理 + if up_weight.shape[0] == param.shape[0] and down_weight.shape[1] == param.shape[1]: + delta = torch.mm(up_weight, down_weight) * lora_strength + param.data += delta + matched_count += 1 + applied_modules.add(full_path) + print(f"成功应用LoRA到 {full_path}.weight") + except Exception as e: + print(f"应用LoRA到 {full_path}.weight 时出错: {str(e)}") + + # 如果上面的直接方法没有匹配到足够数量的参数，则使用递归遍历方法 + if matched_count < min(10, len(common_keys)): + print("直接路径方法匹配数量较少，尝试递归遍历方法...") + + def apply_lora_to_module(module, name_prefix=""): + nonlocal matched_count + + # 如果当前模块路径已经应用过LoRA，则跳过 + if name_prefix in applied_modules: + return + + for name, child in module.named_children(): + full_name = name_prefix + "." + name if name_prefix else name + apply_lora_to_module(child, full_name) + + for param_name, param in module.named_parameters(recurse=False): + if param_name != "weight": + continue + + # 生成各种可能的模块名称匹配 + module_name = name_prefix.split(".")[-1] if name_prefix else "" + compound_name = '.'.join(name_prefix.split(".")[-2:]) if len(name_prefix.split(".")) >= 2 else module_name + + # 如果在transformer块中，尝试取块索引 + block_idx = None + for i, part in enumerate(name_prefix.split(".")): + if part in ["double_blocks", "transformer_blocks", "single_transformer_blocks"] and i+1 < len(name_prefix.split(".")): + block_idx = name_prefix.split(".")[i+1] + break + + block_module = f"{block_idx}.{module_name}" if block_idx else None + + # 尝试不同的匹配方式 + target_key = None + if module_name in common_keys: + target_key = module_name + elif compound_name in common_keys: + target_key = compound_name + elif block_module and block_module in common_keys: + target_key = block_module + + if target_key: + up_key, down_key = up_down_pairs[target_key] + up_weight = lora_sd[up_key] + down_weight = lora_sd[down_key] + + # 尝试不同的矩阵计算方式 + try: + if lora_format == "lycoris": + # 尝试多种矩阵乘法 + delta = None + + # 1. 标准方式 + if down_weight.shape[1] == up_weight.shape[0]: + delta = torch.mm(down_weight, up_weight) * lora_strength + # 2. 反转方式 + elif up_weight.shape[1] == down_weight.shape[0]: + delta = torch.mm(up_weight, down_weight) * lora_strength + # 3. 转置矩阵 + elif up_weight.shape[0] == down_weight.shape[0]: + delta = torch.mm(up_weight.t(), down_weight) * lora_strength + elif up_weight.shape[1] == down_weight.shape[1]: + delta = torch.mm(up_weight, down_weight.t()) * lora_strength + + if delta is not None: + # 如果形状匹配 + if delta.shape == param.shape: + param.data += delta + matched_count += 1 + print(f"成功应用LoRA到 {name_prefix}.{param_name} (匹配键: {target_key})") + # 尝试reshape + else: + try: + delta_reshaped = delta.reshape(param.shape) + param.data += delta_reshaped + matched_count += 1 + print(f"通过reshape成功应用LoRA到 {name_prefix}.{param_name}") + except: + print(f"形状不匹配: {name_prefix}.{param_name} (匹配键: {target_key})") + print(f" 参数形状: {param.shape}, delta形状: {delta.shape}") + else: + # 其他LoRA格式的标准处理 + if up_weight.shape[0] == param.shape[0] and down_weight.shape[1] == param.shape[1]: + delta = torch.mm(up_weight, down_weight) * lora_strength + param.data += delta + matched_count += 1 + print(f"成功应用LoRA到 {name_prefix}.{param_name} (匹配键: {target_key})") + else: + print(f"形状不匹配: {name_prefix}.{param_name} (匹配键: {target_key})") + print(f" 参数形状: {param.shape}, up形状: {up_weight.shape}, down形状: {down_weight.shape}") + except Exception as e: + print(f"应用LoRA到 {name_prefix}.{param_name} 时出错: {str(e)}") + + # 开始递归应用 + apply_lora_to_module(transformer) + + print(f"成功应用了 {matched_count}/{total_pairs} 个LoRA权重对") + return transformer + script_directory = os.path.dirname(os.path.abspath(__file__)) vae_scaling_factor = 0.476986 @@ -101,6 +533,7 @@ def INPUT_TYPES(s): "sageattn", ], {"default": "sdpa"}), "compile_args": ("FRAMEPACKCOMPILEARGS", ), + "lora": ("FRAMEPACKLORA", {"default": None}), } } @@ -110,11 +543,12 @@ def INPUT_TYPES(s): CATEGORY = "FramePackWrapper" def loadmodel(self, model, base_precision, quantization, - compile_args=None, attention_mode="sdpa"): + compile_args=None, attention_mode="sdpa", lora=None): base_dtype = {"fp8_e4m3fn": torch.float8_e4m3fn, "fp8_e4m3fn_fast": torch.float8_e4m3fn, "bf16": torch.bfloat16, "fp16": torch.float16, "fp16_fast": torch.float16, "fp32": torch.float32}[base_precision] device = mm.get_torch_device() + offload_device = mm.unet_offload_device() model_path = os.path.join(folder_paths.models_dir, "diffusers", "lllyasviel", "FramePackI2V_HY") if not os.path.exists(model_path): @@ -127,6 +561,30 @@ def loadmodel(self, model, base_precision, quantization, ) transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained(model_path, torch_dtype=base_dtype, attention_mode=attention_mode).cpu() + + # 创建一个ModelPatcher用于应用LoRA + from comfy import model_patcher + comfy_model = model_patcher.ModelPatcher(transformer, device, offload_device) + + # 加载LoRA + if lora is not None: + from comfy.sd import load_lora_for_models + for l in lora: + print(f"Loading LoRA: {l['name']} with strength: {l['strength']}") + lora_path = l["path"] + lora_strength = l["strength"] + lora_sd = load_torch_file(lora_path, safe_load=True) + lora_sd = standardize_lora_key_format(lora_sd) + + if l["blocks"]: + lora_sd = filter_state_dict_by_blocks(lora_sd, l["blocks"]) + + comfy_model, _ = load_lora_for_models(comfy_model, None, lora_sd, lora_strength, 0) + + # 恢复使用transformer对象 + transformer = comfy_model.model + comfy.model_management.load_models_gpu([comfy_model]) + params_to_keep = {"norm", "bias", "time_in", "vector_in", "guidance_in", "txt_in", "img_in"} if quantization == 'fp8_e4m3fn' or quantization == 'fp8_e4m3fn_fast': transformer = transformer.to(torch.float8_e4m3fn) @@ -173,6 +631,7 @@ def INPUT_TYPES(s): "sageattn", ], {"default": "sdpa"}), "compile_args": ("FRAMEPACKCOMPILEARGS", ), + "lora": ("FRAMEPACKLORA", {"default": None}), } } @@ -182,7 +641,7 @@ def INPUT_TYPES(s): CATEGORY = "FramePackWrapper" def loadmodel(self, model, base_precision, quantization, - compile_args=None, attention_mode="sdpa"): + compile_args=None, attention_mode="sdpa", lora=None): base_dtype = {"fp8_e4m3fn": torch.float8_e4m3fn, "fp8_e4m3fn_fast": torch.float8_e4m3fn, "bf16": torch.bfloat16, "fp16": torch.float16, "fp16_fast": torch.float16, "fp32": torch.float32}[base_precision] @@ -216,6 +675,29 @@ def loadmodel(self, model, base_precision, quantization, set_module_tensor_to_device(transformer, name, device=offload_device, dtype=dtype_to_use, value=sd[name]) + # 创建一个ModelPatcher用于应用LoRA + from comfy import model_patcher + comfy_model = model_patcher.ModelPatcher(transformer, device, offload_device) + + # 加载LoRA + if lora is not None: + from comfy.sd import load_lora_for_models + for l in lora: + print(f"Loading LoRA: {l['name']} with strength: {l['strength']}") + lora_path = l["path"] + lora_strength = l["strength"] + lora_sd = load_torch_file(lora_path, safe_load=True) + lora_sd = standardize_lora_key_format(lora_sd) + + if l["blocks"]: + lora_sd = filter_state_dict_by_blocks(lora_sd, l["blocks"]) + + comfy_model, _ = load_lora_for_models(comfy_model, None, lora_sd, lora_strength, 0) + + # 恢复使用transformer对象 + transformer = comfy_model.model + comfy.model_management.load_models_gpu([comfy_model]) + if quantization == "fp8_e4m3fn_fast": from .fp8_optimization import convert_fp8_linear convert_fp8_linear(transformer, base_dtype, params_to_keep=params_to_keep) @@ -239,6 +721,104 @@ def loadmodel(self, model, base_precision, quantization, } return (pipe, ) +class LoadFramePackModelWithLoRA: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": (folder_paths.get_filename_list("diffusion_models"), {"tooltip": "These models are loaded from the 'ComfyUI/models/diffusion_models' -folder",}), + + "base_precision": (["fp32", "bf16", "fp16"], {"default": "bf16"}), + "quantization": (['disabled', 'fp8_e4m3fn', 'fp8_e4m3fn_fast', 'fp8_e5m2'], {"default": 'disabled', "tooltip": "optional quantization method"}), + }, + "optional": { + "attention_mode": ([ + "sdpa", + "flash_attn", + "sageattn", + ], {"default": "sdpa"}), + "compile_args": ("FRAMEPACKCOMPILEARGS", ), + "lora": ("FRAMEPACKLORA", {"default": None}), + } + } + + RETURN_TYPES = ("FramePackMODEL",) + RETURN_NAMES = ("model", ) + FUNCTION = "loadmodel" + CATEGORY = "FramePackWrapper" + DESCRIPTION = "加载FramePack模型并使用手动方法应用LoRA" + + def loadmodel(self, model, base_precision, quantization, + compile_args=None, attention_mode="sdpa", lora=None): + + base_dtype = {"fp8_e4m3fn": torch.float8_e4m3fn, "fp8_e4m3fn_fast": torch.float8_e4m3fn, "bf16": torch.bfloat16, "fp16": torch.float16, "fp16_fast": torch.float16, "fp32": torch.float32}[base_precision] + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + model_path = folder_paths.get_full_path_or_raise("diffusion_models", model) + model_config_path = os.path.join(script_directory, "transformer_config.json") + import json + with open(model_config_path, "r") as f: + config = json.load(f) + sd = load_torch_file(model_path, device=offload_device, safe_load=True) + + with init_empty_weights(): + transformer = HunyuanVideoTransformer3DModelPacked(**config, attention_mode=attention_mode) + + params_to_keep = {"norm", "bias", "time_in", "vector_in", "guidance_in", "txt_in", "img_in"} + if quantization == "fp8_e4m3fn" or quantization == "fp8_e4m3fn_fast" or quantization == "fp8_scaled": + dtype = torch.float8_e4m3fn + elif quantization == "fp8_e5m2": + dtype = torch.float8_e5m2 + else: + dtype = base_dtype + print("Using accelerate to load and assign model weights to device...") + param_count = sum(1 for _ in transformer.named_parameters()) + for name, param in tqdm(transformer.named_parameters(), + desc=f"Loading transformer parameters to {offload_device}", + total=param_count, + leave=True): + dtype_to_use = base_dtype if any(keyword in name for keyword in params_to_keep) else dtype + + set_module_tensor_to_device(transformer, name, device=offload_device, dtype=dtype_to_use, value=sd[name]) + + # 使用手动方法应用LoRA，避免使用ModelPatcher + if lora is not None: + for l in lora: + print(f"Loading LoRA: {l['name']} with strength: {l['strength']}") + lora_path = l["path"] + lora_strength = l["strength"] + lora_sd = load_torch_file(lora_path, safe_load=True) + lora_sd = standardize_lora_key_format(lora_sd) + + if l["blocks"]: + lora_sd = filter_state_dict_by_blocks(lora_sd, l["blocks"]) + + # 使用我们的自定义函数手动应用LoRA + transformer = apply_lora_weights_manually(transformer, lora_sd, lora_strength) + + if quantization == "fp8_e4m3fn_fast": + from .fp8_optimization import convert_fp8_linear + convert_fp8_linear(transformer, base_dtype, params_to_keep=params_to_keep) + + + DynamicSwapInstaller.install_model(transformer, device=device) + + if compile_args is not None: + if compile_args["compile_single_blocks"]: + for i, block in enumerate(transformer.single_transformer_blocks): + transformer.single_transformer_blocks[i] = torch.compile(block, fullgraph=compile_args["fullgraph"], dynamic=compile_args["dynamic"], backend=compile_args["backend"], mode=compile_args["mode"]) + if compile_args["compile_double_blocks"]: + for i, block in enumerate(transformer.transformer_blocks): + transformer.transformer_blocks[i] = torch.compile(block, fullgraph=compile_args["fullgraph"], dynamic=compile_args["dynamic"], backend=compile_args["backend"], mode=compile_args["mode"]) + + pipe = { + "transformer": transformer.eval(), + "dtype": base_dtype, + } + return (pipe, ) + class FramePackFindNearestBucket: @classmethod def INPUT_TYPES(s): @@ -493,6 +1073,69 @@ def process(self, model, shift, positive, negative, latent_window_size, use_teac mm.soft_empty_cache() return {"samples": real_history_latents / vae_scaling_factor}, +class FramePackLoraBlockEdit: + def __init__(self): + self.loaded_lora = None + + @classmethod + def INPUT_TYPES(s): + arg_dict = {} + argument = ("BOOLEAN", {"default": True}) + + for i in range(20): + arg_dict[f"transformer_blocks.{i}."] = argument + + for i in range(40): + arg_dict[f"single_transformer_blocks.{i}."] = argument + + return {"required": arg_dict} + + RETURN_TYPES = ("SELECTEDBLOCKS", ) + RETURN_NAMES = ("blocks", ) + FUNCTION = "select" + CATEGORY = "FramePackWrapper" + DESCRIPTION = "选择要应用LoRA的模型块" + + def select(self, **kwargs): + selected_blocks = {k: v for k, v in kwargs.items() if v is True} + print("Selected blocks: ", selected_blocks) + return (selected_blocks,) + +class FramePackLoraSelect: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "lora": (folder_paths.get_filename_list("loras"), + {"tooltip": "LORA模型应位于ComfyUI/models/loras目录，扩展名为.safetensors"}), + "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.0001, "tooltip": "LoRA的强度，设置为0可卸载LoRA"}), + }, + "optional": { + "prev_lora":("FRAMEPACKLORA", {"default": None, "tooltip": "用于加载多个LoRA"}), + "blocks":("SELECTEDBLOCKS", ), + } + } + + RETURN_TYPES = ("FRAMEPACKLORA",) + RETURN_NAMES = ("lora", ) + FUNCTION = "getlorapath" + CATEGORY = "FramePackWrapper" + DESCRIPTION = "从ComfyUI/models/loras选择LoRA模型" + + def getlorapath(self, lora, strength, blocks=None, prev_lora=None): + loras_list = [] + + lora = { + "path": folder_paths.get_full_path("loras", lora), + "strength": strength, + "name": lora.split(".")[0], + "blocks": blocks + } + if prev_lora is not None: + loras_list.extend(prev_lora) + + loras_list.append(lora) + return (loras_list,) NODE_CLASS_MAPPINGS = { "DownloadAndLoadFramePackModel": DownloadAndLoadFramePackModel, @@ -500,6 +1143,9 @@ def process(self, model, shift, positive, negative, latent_window_size, use_teac "FramePackTorchCompileSettings": FramePackTorchCompileSettings, "FramePackFindNearestBucket": FramePackFindNearestBucket, "LoadFramePackModel": LoadFramePackModel, + "FramePackLoraBlockEdit": FramePackLoraBlockEdit, + "FramePackLoraSelect": FramePackLoraSelect, + "LoadFramePackModelWithLoRA": LoadFramePackModelWithLoRA, } NODE_DISPLAY_NAME_MAPPINGS = { "DownloadAndLoadFramePackModel": "(Down)Load FramePackModel", @@ -507,5 +1153,8 @@ def process(self, model, shift, positive, negative, latent_window_size, use_teac "FramePackTorchCompileSettings": "Torch Compile Settings", "FramePackFindNearestBucket": "Find Nearest Bucket", "LoadFramePackModel": "Load FramePackModel", + "FramePackLoraBlockEdit": "FramePack LoRA Block Edit", + "FramePackLoraSelect": "FramePack LoRA Select", + "LoadFramePackModelWithLoRA": "Load FramePackModel With LoRA", }