IP-Adapter compile error

[songh11]

Describe the bug

Use onediff compile with IP-Adapter had some error.

Your environment

Ubuntu 20.04
torch 2.1.2
diffusers 0.27.0

OneDiff git commit id

dev_support_diffusers_ipa branch

OneFlow version info if you have installed oneflow

version: 0.9.1.dev20240703+cu121
git_commit: ec7b682
cmake_build_type: Release
rdma: True
mlir: True
enterprise: False

How To Reproduce

Steps to reproduce the behavior(code or script):

from diffusers import StableDiffusionXLPipeline

import torch
from PIL import Image
from onediffx import compile_pipe
from onediff.infer_compiler import oneflow_compile

from ip_adapter.ip_adapter import IPAdapterXL

model_path = "stabilityai/stable-diffusion-xl-base-1.0"
ip_adapter_path = "h94/IP-Adapter"
device = "cuda"
image = Image.open("./images/girl.png")

pipe = StableDiffusionXLPipeline.from_pretrained(
    model_path,
    torch_dtype=torch.float16,
    variant='fp16',
    use_safetensors=True,
).to('cuda')
pipe.load_ip_adapter(ip_adapter_path, subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")


generator = torch.Generator(device=device).manual_seed(0)
pipe.set_ip_adapter_scale(0.4)
images = pipe(
    prompt="PLUS (high strength)",
    ip_adapter_image=image,
    negative_prompt="deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality",
    num_inference_steps=30,
    generator=generator,
).images
images[0].save(f"./images/ip_adapter_out.png")

pipe.unet = oneflow_compile(pipe.unet)
generator = torch.Generator(device=device).manual_seed(0)

images = pipe(
    prompt="PLUS (high strength)",
    ip_adapter_image=image,
    negative_prompt="deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality",
    num_inference_steps=30,
    generator=generator,
).images
images[0].save(f"./images/ip_adapter_compile_out.png")

The complete error message

Details

Traceback (most recent call last):
File "/home/sh9/workspace/onediff/onediff_diffusers_extensions/examples/image_to_image_ip_adapter.py", line 37, in
images = pipe(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py", line 1174, in call
noise_pred = self.unet(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1568, in _call_impl
result = forward_call(*args, **kwargs)
File "/home/sh9/workspace/onediff/src/onediff/infer_compiler/backends/oneflow/online_quantization_utils.py", line 61, in wrapper
output = func(self, *args, **kwargs)
File "/home/sh9/workspace/onediff/src/onediff/infer_compiler/backends/oneflow/args_tree_util.py", line 52, in wrapper
output = func(self, *mapped_args, **mapped_kwargs)
File "/home/sh9/workspace/onediff/src/onediff/infer_compiler/backends/oneflow/deployable_module.py", line 41, in wrapper
return func(self, *args, **kwargs)
File "/home/sh9/workspace/onediff/src/onediff/infer_compiler/backends/oneflow/graph_management_utils.py", line 121, in wrapper
ret = func(self, *args, **kwargs)
File "/home/sh9/workspace/onediff/src/onediff/infer_compiler/backends/oneflow/deployable_module.py", line 137, in forward
output = dpl_graph(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/graph.py", line 295, in call
self._compile(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/graph.py", line 861, in _compile
return self._dynamic_input_graph_cache._compile(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/cache.py", line 121, in _compile
return graph._compile(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/graph.py", line 865, in _compile
return self._compile_new(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/graph.py", line 884, in _compile_new
_, eager_outputs = self.build_graph(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/graph.py", line 1429, in build_graph
outputs = self.__build_graph(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/graph.py", line 1577, in __build_graph
outputs = self.build(*lazy_args, **lazy_kwargs)
File "/home/sh9/workspace/onediff/src/onediff/infer_compiler/backends/oneflow/graph.py", line 19, in build
return self.model(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 188, in call
result = self.__block_forward(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 238, in __block_forward
result = unbound_forward_of_module_instance(self, *args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/diffusers/models/unets/unet_2d_condition.py", line 1219, in forward
sample, res_samples = downsample_block(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 188, in call
result = self.__block_forward(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 238, in __block_forward
result = unbound_forward_of_module_instance(self, *args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/diffusers/models/unets/unet_2d_blocks.py", line 1279, in forward
hidden_states = attn(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 188, in call
result = self.__block_forward(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 238, in __block_forward
result = unbound_forward_of_module_instance(self, *args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/diffusers/models/transformers/transformer_2d.py", line 397, in forward
hidden_states = block(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 188, in call
result = self.__block_forward(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 238, in __block_forward
result = unbound_forward_of_module_instance(self, *args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/diffusers/models/attention.py", line 366, in forward
attn_output = self.attn2(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 188, in call
result = self.__block_forward(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 238, in __block_forward
result = unbound_forward_of_module_instance(self, *args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/diffusers/models/attention_processor.py", line 522, in forward
return self.processor(
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 188, in call
result = self.__block_forward(*args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/graph/proxy.py", line 238, in __block_forward
result = unbound_forward_of_module_instance(self, *args, **kwargs)
File "/home/sh9/.virtualenvs/oneflow_py310/lib/python3.10/site-packages/oneflow/nn/modules/module.py", line 200, in forward
raise NotImplementedError()
NotImplementedError

Additional context

Add any other context about the problem here.

[songh11]

Using this demo is still problem: #837

[zhangvia]

maybe you should reinstall all the onediff stuff,i run ipadapter compile using this branch successfully. but still get those problem:
(1) different resolution will trigger recompile
(2) pipe.set_ip_adapter_scale() won't work, the scale never change

[songh11]

maybe you should reinstall all the onediff stuff,i run ipadapter compile using this branch successfully. but still get those problem: (1) different resolution will trigger recompile (2) pipe.set_ip_adapter_scale() won't work, the scale never change

感谢回复，好像是这里的问题，我这边也可以跑了，

[zhangvia]

感谢回复，好像是这里的问题，我这边也可以跑了，

你的报错里确实体现了使用的是diffusers原本的模块，onediff在infer_compiler_registry中有重新实现绝大部分attention_processor。但是我说的两个问题还是没有解决