chore: bump open-clip-torch to v2.8.0

server/clip_server/model/model.py

@@ -19,7 +19,7 @@
 from open_clip.transformer import QuickGELU, LayerNorm, LayerNormFp32, Attention
 from open_clip.timm_model import TimmModel
 from open_clip.factory import _MODEL_CONFIGS
-from open_clip.hf_model import PreTrainedTextEncoder
+from open_clip.hf_model import HFTextEncoder
 from open_clip.transformer import ResidualAttentionBlock as _ResidualAttentionBlock
 from open_clip.transformer import Transformer as _Transformer
 from open_clip.transformer import VisionTransformer as _VisionTransformer
@@ -75,11 +75,20 @@ def __init__(
         self,
         image_size: int,
         patch_size: int,
+        global_average_pool: bool,
         output_dim: int,
+        patch_dropout: float,
         dtype: torch.dtype = torch.float32,
         **kwargs,
     ):
-        super().__init__(image_size, patch_size, output_dim=output_dim, **kwargs)
+        super().__init__(
+            image_size,
+            patch_size,
+            global_average_pool=global_average_pool,
+            output_dim=output_dim,
+            patch_dropout=patch_dropout,
+            **kwargs,
+        )
         self.transformer = Transformer(dtype=dtype, **kwargs)
 
     def forward(self, x: torch.Tensor):
@@ -111,6 +120,8 @@ class CLIPVisionCfg:
     patch_size: int = 16
     image_size: Union[Tuple[int, int], int] = 224
     ls_init_value: Optional[float] = None  # layer scale initial value
+    patch_dropout: float = 0.0  # what fraction of patches to dropout during training (0 would mean disabled and no patches dropped) - 0.5 to 0.75 recommended in the paper for optimal results
+    global_average_pool: bool = False  # whether to global average pool the last embedding layer, instead of using CLS token (https://arxiv.org/abs/2205.01580)
     timm_model_name: str = (
         None  # a valid model name overrides layers, width, patch_size
     )
@@ -136,6 +147,7 @@ class CLIPTextCfg:
     ls_init_value: Optional[float] = None  # layer scale initial value
     hf_model_name: str = None
     hf_tokenizer_name: str = None
+    hf_model_pretrained: bool = True
     proj: str = 'mlp'
     pooler_type: str = 'mean_pooler'
 
@@ -189,6 +201,8 @@ def _build_vision_tower(
             heads=vision_heads,
             mlp_ratio=vision_cfg.mlp_ratio,
             ls_init_value=vision_cfg.ls_init_value,
+            patch_dropout=vision_cfg.patch_dropout,
+            global_average_pool=vision_cfg.global_average_pool,
             output_dim=embed_dim,
             act_layer=act_layer,
             norm_layer=norm_layer,
@@ -208,11 +222,12 @@ def _build_text_tower(
         text_cfg = CLIPTextCfg(**text_cfg)
 
     if text_cfg.hf_model_name:
-        text = PreTrainedTextEncoder(
+        text = HFTextEncoder(
             text_cfg.hf_model_name,
             output_dim=embed_dim,
             proj=text_cfg.proj,
             pooler_type=text_cfg.pooler_type,
+            pretrained=text_cfg.hf_model_pretrained,
         )
     else:
         act_layer = QuickGELU if quick_gelu else nn.GELU
@@ -555,6 +570,7 @@ def load_openclip_model(
     jit: bool = False,
     force_quick_gelu: bool = False,
     force_custom_text: bool = False,
+    force_patch_dropout: Optional[float] = None,
     pretrained_image: bool = False,
     dtype: Optional[Union[str, torch.dtype]] = None,
 ):
@@ -578,6 +594,10 @@ def load_openclip_model(
         # override for use of QuickGELU on non-OpenAI transformer models
         model_cfg["quick_gelu"] = True
 
+    if force_patch_dropout is not None:
+        # override the default patch dropout value
+        model_cfg["vision_cfg"]["patch_dropout"] = force_patch_dropout
+
     if pretrained_image:
         if 'timm_model_name' in model_cfg.get('vision_cfg', {}):
             # pretrained weight loading for timm models set via vision_cfg

server/setup.py

@@ -47,7 +47,7 @@
         'torchvision<=0.13.0' if sys.version_info <= (3, 7, 2) else 'torchvision',
         'jina>=3.12.0',
         'prometheus-client',
-        'open_clip_torch>=2.7.0',
+        'open_clip_torch>=2.8.0',
     ],
     extras_require={
         'onnx': [