Implement Differential Diffusion
diff --git a/comfy/samplers.py b/comfy/samplers.py
index c795f208d8..a4a5595117 100644
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -276,6 +276,8 @@ def __init__(self, model):
         self.inner_model = model
     def forward(self, x, sigma, uncond, cond, cond_scale, denoise_mask, model_options={}, seed=None):
         if denoise_mask is not None:
+            if "denoise_mask_function" in model_options:
+                denoise_mask = model_options["denoise_mask_function"](sigma, denoise_mask)
             latent_mask = 1. - denoise_mask
             x = x * denoise_mask + (self.latent_image + self.noise * sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1))) * latent_mask
         out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, model_options=model_options, seed=seed)
diff --git a/comfy_extras/nodes_differential_diffusion.py b/comfy_extras/nodes_differential_diffusion.py
new file mode 100644
index 0000000000..48c95602ff
--- /dev/null
+++ b/comfy_extras/nodes_differential_diffusion.py
@@ -0,0 +1,97 @@
+# code adapted from https://github.com/exx8/differential-diffusion
+
+import torch
+import inspect
+
+class DifferentialDiffusion():
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"model": ("MODEL", ),
+                            }}
+    RETURN_TYPES = ("MODEL",)
+    FUNCTION = "apply"
+    CATEGORY = "_for_testing"
+    INIT = False
+
+    @classmethod
+    def IS_CHANGED(s, *args, **kwargs):
+        DifferentialDiffusion.INIT = s.INIT = True
+        return ""
+
+    def __init__(self) -> None:
+        DifferentialDiffusion.INIT = False
+        self.sigmas: torch.Tensor = None
+        self.thresholds: torch.Tensor = None
+        self.mask_i = None
+        self.valid_sigmas = False
+        self.varying_sigmas_samplers = ["dpmpp_2s", "dpmpp_sde", "dpm_2", "heun", "restart"]
+
+    def apply(self, model):
+        model = model.clone()
+        model.model_options["denoise_mask_function"] = self.forward
+        return (model,)
+    
+    def init_sigmas(self, sigma: torch.Tensor, denoise_mask: torch.Tensor):
+        self.__init__()
+        self.sigmas, sampler = find_outer_instance("sigmas", callback=get_sigmas_and_sampler) or (None, "")
+        self.valid_sigmas = not ("sample_" not in sampler or any(s in sampler for s in self.varying_sigmas_samplers)) or "generic" in sampler
+        if self.sigmas is None:
+            self.sigmas = sigma[:1].repeat(2)
+            self.sigmas[-1].zero_()
+        self.sigmas_min = self.sigmas.min()
+        self.sigmas_max = self.sigmas.max()
+        self.thresholds = torch.linspace(1, 0, self.sigmas.shape[0], dtype=sigma.dtype, device=sigma.device)
+        self.thresholds_min_len = self.thresholds.shape[0] - 1
+        if self.valid_sigmas:
+            thresholds = self.thresholds[:-1].reshape(-1, 1, 1, 1, 1)
+            mask = denoise_mask.unsqueeze(0)
+            mask = (mask >= thresholds).to(denoise_mask.dtype)
+            self.mask_i = iter(mask)
+    
+    def forward(self, sigma: torch.Tensor, denoise_mask: torch.Tensor):
+        if self.sigmas is None or DifferentialDiffusion.INIT:
+            self.init_sigmas(sigma, denoise_mask)
+        if self.valid_sigmas:
+            try:
+                return next(self.mask_i)
+            except StopIteration:
+                self.valid_sigmas = False
+        if self.thresholds_min_len > 1:
+            nearest_idx = (self.sigmas - sigma[0]).abs().argmin()
+            if not self.thresholds_min_len > nearest_idx:
+                nearest_idx = -2
+            threshold = self.thresholds[nearest_idx]
+        else:
+            threshold = (sigma[0] - self.sigmas_min) / (self.sigmas_max - self.sigmas_min)
+        return (denoise_mask >= threshold).to(denoise_mask.dtype)
+
+def get_sigmas_and_sampler(frame, target):
+    found = frame.f_locals[target]
+    if isinstance(found, torch.Tensor) and found[-1] < 0.1:
+        return found, frame.f_code.co_name
+    return False
+
+def find_outer_instance(target: str, target_type=None, callback=None):
+    frame = inspect.currentframe()
+    i = 0
+    while frame and i < 100:
+        if target in frame.f_locals:
+            if callback is not None:
+                res = callback(frame, target)
+                if res:
+                    return res
+            else:
+                found = frame.f_locals[target]
+                if isinstance(found, target_type):
+                    return found
+        frame = frame.f_back
+        i += 1
+    return None
+
+    
+NODE_CLASS_MAPPINGS = {
+    "DifferentialDiffusion": DifferentialDiffusion,
+}
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "DifferentialDiffusion": "Differential Diffusion",
+}
diff --git a/nodes.py b/nodes.py
index a577c21262..b759d22cef 100644
--- a/nodes.py
+++ b/nodes.py
@@ -1961,6 +1961,7 @@ def init_custom_nodes():
         "nodes_photomaker.py",
         "nodes_cond.py",
         "nodes_stable_cascade.py",
+        "nodes_differential_diffusion.py",
     ]
 
     for node_file in extras_files:

The authors of the paper introduced a concept called a _change map_. For brevity, I will refer to it here as a _mask_.

The node `_for_testing/Differential Diffusion` is meant to be used alongside the `InpaintModelConditioning` node or the `Set Latent Noise Mask` node, and then the sampler does it automatically.

#### Implementation details
* A [full change](https://github.com/exx8/differential-diffusion/issues/2#issuecomment-1738136217) of the mask is done, essentially $\fbox{mask >= threshold}$ instead of $\fbox{mask > threshold}$. This is important because the contents of the mask should be followed, and for latent inpainting/outpainting (where the mask becomes binary), the initial step _should_ paint _something_.


#### General info/tips
* Lighter areas get painted earlier, darker areas later.
    * Try adjusting the mask's blur, opacity, brightness, exposure, etc.
    * To get more of the original image, lower the mask's opacity/brightness or increase its blur.
* The more steps, the finer it's applied (due to finer thresholds).
* Lower your cfg if things look too noisy or fried.

#### Comparisons

These were made alongside the `InpaintModelConditioning` node _without_ an inpainting model.

|    |  
| :----: | 
| ![ComfyUI_temp_vytsl_00046_](https://github.com/comfyanonymous/ComfyUI/assets/54494639/70becbb2-18b9-4307-837d-903018e36258) |  

|  |  |
| :----: |  :----: | 
| <img height="512" alt="Example comparison 2" src="https://github.com/comfyanonymous/ComfyUI/assets/54494639/c593ccdc-4078-4e89-bb9f-d0611b23d2d3"> | <img height="512" alt="Example comparison 3" src="https://github.com/comfyanonymous/ComfyUI/assets/54494639/180061cf-2f69-4455-9ec6-17f1d2c4038e"> |





#### Example workflow  
Here's a simple workflow to get started. (Note: It uses some custom nodes).
Feel free to use another blur node.
Also, see the official [inpaint examples](https://comfyanonymous.github.io/ComfyUI_examples/inpaint/).

<div align="center">
    <img width="256" alt="Example workflow" src="https://github.com/shiimizu/ComfyUI_smZNodes/assets/https://github.com/comfyanonymous/ComfyUI/assets/54494639/5a9b143c-2b48-4d87-83a5-15e9448367df">
    <p>Download this image to drag & drop it in ComfyUI.</p>
</div>


#### Issues
* ~~The callback doesn't seem to get called for UniPC samplers.~~

---

Resolves #2851, resolves #2671 