Add Unified-Bench old protocol bundle: unified_bench_old_protocol_0p8346/scripts/scorers

unified_bench_old_protocol_0p8346/scripts/scorers/__init__.py

@@ -0,0 +1,34 @@
+"""
+Similarity scorers for Unified-Bench.
+
+Each scorer is a class with:
+  - __init__(model_path: str, device: str = "cuda")
+  - score_pair(ref_image, gen_image) -> float
+  - encode_image(image) -> torch.Tensor  (optional, enables batch scoring)
+
+All scorers return cosine similarity in [-1, 1], higher is better.
+
+Ported from model/UAE/Unified-Bench/{CLIP,DINO_v2,DINO_v3,LongCLIP}.py with
+hardcoded paths removed (paths now come from the eval config).
+"""
+from __future__ import annotations
+
+from typing import Any, Dict
+
+
+def load_scorer(name: str, model_path: str, device: str = "cuda"):
+    """Factory: instantiate a scorer by name. Raises ImportError if deps missing."""
+    name_lower = name.strip().lower()
+    if name_lower == "clip":
+        from .clip_scorer import CLIPScorer
+        return CLIPScorer(model_path=model_path, device=device)
+    if name_lower in ("dinov2", "dino_v2", "dino-v2"):
+        from .dinov2_scorer import DINOv2Scorer
+        return DINOv2Scorer(model_path=model_path, device=device)
+    if name_lower in ("dinov3", "dino_v3", "dino-v3"):
+        from .dinov3_scorer import DINOv3Scorer
+        return DINOv3Scorer(model_path=model_path, device=device)
+    if name_lower in ("longclip", "long_clip", "long-clip"):
+        from .longclip_scorer import LongCLIPScorer
+        return LongCLIPScorer(model_path=model_path, device=device)
+    raise ValueError(f"Unknown scorer '{name}'. Supported: clip, dinov2, dinov3, longclip")

unified_bench_old_protocol_0p8346/scripts/scorers/clip_scorer.py

@@ -0,0 +1,50 @@
+"""CLIP vision-encoder similarity (ported from model/UAE/Unified-Bench/CLIP.py).
+
+Uses CLIPImageProcessor directly (not CLIPProcessor) and explicitly moves
+pixel_values to the target device. This avoids issues with BatchFeature.to()
+on newer transformers versions where passing only images through the combined
+CLIPProcessor can emit deprecation warnings / silently break.
+"""
+from __future__ import annotations
+
+import torch
+from torch.nn import functional as F
+from PIL import Image
+from transformers import CLIPImageProcessor, CLIPModel
+
+
+class CLIPScorer:
+    def __init__(self, model_path: str = "openai/clip-vit-large-patch14", device: str = "cuda"):
+        self.device = torch.device(device if torch.cuda.is_available() and device == "cuda" else "cpu")
+        self.model_path = model_path
+        print(f"[clip] loading from {model_path}")
+        self.model = CLIPModel.from_pretrained(model_path).to(self.device)
+        self.model.eval()
+        self.image_processor = CLIPImageProcessor.from_pretrained(model_path)
+        self.model_dtype = next(self.model.parameters()).dtype
+
+    def _to_pil(self, image):
+        if isinstance(image, str):
+            return Image.open(image).convert("RGB")
+        if isinstance(image, Image.Image):
+            return image.convert("RGB") if image.mode != "RGB" else image
+        raise ValueError(f"CLIPScorer expects PIL.Image or path, got {type(image)}")
+
+    @torch.no_grad()
+    def encode_image(self, image):
+        pil = self._to_pil(image)
+        processed = self.image_processor(images=pil, return_tensors="pt")
+        pixel_values = processed["pixel_values"].to(device=self.device, dtype=self.model_dtype)
+        # Bypass CLIPModel.get_image_features — in recent transformers versions
+        # it can return the full BaseModelOutputWithPooling dataclass instead
+        # of the projected features tensor. Call vision_model + visual_projection
+        # directly to stay stable across versions.
+        vision_out = self.model.vision_model(pixel_values=pixel_values)
+        pooled = vision_out.pooler_output  # (batch, hidden_size)
+        features = self.model.visual_projection(pooled)  # (batch, projection_dim)
+        return F.normalize(features, p=2, dim=1)
+
+    def score_pair(self, ref_image, gen_image) -> float:
+        f1 = self.encode_image(ref_image)
+        f2 = self.encode_image(gen_image)
+        return torch.cosine_similarity(f1, f2, dim=1).item()

unified_bench_old_protocol_0p8346/scripts/scorers/dinov2_scorer.py

@@ -0,0 +1,38 @@
+"""DINOv2 CLS-token similarity (ported from model/UAE/Unified-Bench/DINO_v2.py)."""
+from __future__ import annotations
+
+import torch
+from torch.nn import functional as F
+from PIL import Image
+from transformers import AutoImageProcessor, AutoModel
+
+
+class DINOv2Scorer:
+    def __init__(self, model_path: str = "facebook/dinov2-large", device: str = "cuda"):
+        self.device = torch.device(device if torch.cuda.is_available() and device == "cuda" else "cpu")
+        self.model_path = model_path
+        print(f"[dinov2] loading from {model_path}")
+        self.processor = AutoImageProcessor.from_pretrained(model_path)
+        self.model = AutoModel.from_pretrained(model_path).to(self.device)
+        self.model.eval()
+
+    def _to_pil(self, image):
+        if isinstance(image, str):
+            return Image.open(image).convert("RGB")
+        if isinstance(image, Image.Image):
+            return image.convert("RGB") if image.mode != "RGB" else image
+        raise ValueError(f"DINOv2Scorer expects PIL.Image or path, got {type(image)}")
+
+    @torch.no_grad()
+    def encode_image(self, image):
+        pil = self._to_pil(image)
+        inputs = self.processor(images=pil, return_tensors="pt").to(self.device)
+        outputs = self.model(**inputs)
+        # CLS token
+        features = outputs.last_hidden_state[:, 0, :]
+        return F.normalize(features, p=2, dim=1)
+
+    def score_pair(self, ref_image, gen_image) -> float:
+        f1 = self.encode_image(ref_image)
+        f2 = self.encode_image(gen_image)
+        return torch.cosine_similarity(f1, f2, dim=1).item()

unified_bench_old_protocol_0p8346/scripts/scorers/dinov3_scorer.py

@@ -0,0 +1,43 @@
+"""DINOv3 CLS-token similarity (ported from model/UAE/Unified-Bench/DINO_v3.py).
+
+Requires transformers >= 4.50 with DINOv3 support, or the HF repo contents to load
+via AutoModel with trust_remote_code.
+"""
+from __future__ import annotations
+
+import torch
+from torch.nn import functional as F
+from PIL import Image
+from transformers import AutoImageProcessor, AutoModel
+
+
+class DINOv3Scorer:
+    def __init__(self, model_path: str, device: str = "cuda"):
+        self.device = torch.device(device if torch.cuda.is_available() and device == "cuda" else "cpu")
+        self.model_path = model_path
+        print(f"[dinov3] loading from {model_path}")
+        # DINOv3 checkpoints sometimes use custom modeling code; trust_remote_code
+        # keeps us compatible with BAAI/Meta releases that ship model.py alongside weights.
+        self.processor = AutoImageProcessor.from_pretrained(model_path, trust_remote_code=True)
+        self.model = AutoModel.from_pretrained(model_path, trust_remote_code=True).to(self.device)
+        self.model.eval()
+
+    def _to_pil(self, image):
+        if isinstance(image, str):
+            return Image.open(image).convert("RGB")
+        if isinstance(image, Image.Image):
+            return image.convert("RGB") if image.mode != "RGB" else image
+        raise ValueError(f"DINOv3Scorer expects PIL.Image or path, got {type(image)}")
+
+    @torch.no_grad()
+    def encode_image(self, image):
+        pil = self._to_pil(image)
+        inputs = self.processor(images=pil, return_tensors="pt").to(self.device)
+        outputs = self.model(**inputs)
+        features = outputs.last_hidden_state[:, 0, :]
+        return F.normalize(features, p=2, dim=1)
+
+    def score_pair(self, ref_image, gen_image) -> float:
+        f1 = self.encode_image(ref_image)
+        f2 = self.encode_image(gen_image)
+        return torch.cosine_similarity(f1, f2, dim=1).item()

unified_bench_old_protocol_0p8346/scripts/scorers/longclip_scorer.py

@@ -0,0 +1,62 @@
+"""LongCLIP similarity (ported from model/UAE/Unified-Bench/LongCLIP.py).
+
+LongCLIP weights are a .pt file from BeichenZhang/Long-CLIP. The repo is NOT
+a pip package, so it's cloned into /opt/longclip at image build time (see
+modal/images.py:_unified_bench_image). We add /opt to sys.path and import
+`from longclip.model import longclip`.
+
+If the import fails at runtime, this scorer raises ImportError and the
+scoring script will skip it with a warning.
+"""
+from __future__ import annotations
+
+import os
+import sys
+
+import torch
+from torch.nn import functional as F
+from PIL import Image
+
+
+class LongCLIPScorer:
+    def __init__(self, model_path: str, device: str = "cuda"):
+        self.device = torch.device(device if torch.cuda.is_available() and device == "cuda" else "cpu")
+        self.model_path = model_path
+        print(f"[longclip] loading from {model_path}")
+
+        # Allow overriding the clone location via env var, default /opt.
+        longclip_root = os.environ.get("LONGCLIP_ROOT", "/opt")
+        if longclip_root not in sys.path:
+            sys.path.insert(0, longclip_root)
+
+        try:
+            from longclip.model import longclip
+        except ImportError as exc:
+            raise ImportError(
+                f"LongCLIP module not found at {longclip_root}/longclip. "
+                "Expected repo cloned from https://github.com/beichenzbc/Long-CLIP "
+                "(done in modal/images.py:_unified_bench_image)."
+            ) from exc
+
+        self._longclip = longclip
+        self.model, self.tform = longclip.load(model_path, device=self.device)
+        self.model.eval()
+
+    def _to_pil(self, image):
+        if isinstance(image, str):
+            return Image.open(image).convert("RGB")
+        if isinstance(image, Image.Image):
+            return image.convert("RGB") if image.mode != "RGB" else image
+        raise ValueError(f"LongCLIPScorer expects PIL.Image or path, got {type(image)}")
+
+    @torch.no_grad()
+    def encode_image(self, image):
+        pil = self._to_pil(image)
+        tensor = self.tform(pil).unsqueeze(0).to(device=self.device, dtype=self.model.dtype)
+        features = self.model.encode_image(tensor)
+        return F.normalize(features, p=2, dim=1)
+
+    def score_pair(self, ref_image, gen_image) -> float:
+        f1 = self.encode_image(ref_image)
+        f2 = self.encode_image(gen_image)
+        return torch.cosine_similarity(f1, f2, dim=1).item()