-
Notifications
You must be signed in to change notification settings - Fork 0
/
clip_models.py
49 lines (36 loc) · 1.49 KB
/
clip_models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import torch
from torch import nn
from models import open_clip_lora
import logging as python_logging
python_logging.basicConfig(level=python_logging.INFO)
class CLIPWrapper(nn.Module):
def __init__(self,
backbone: str,
pretrained_path: str,
device: str):
"""
:param backbone:
:param pretrained_path
:param device:
"""
super().__init__()
self.device = device
# open_clip case
self.model, _, self.image_preprocess = open_clip_lora.create_model_and_transforms(backbone,
device=self.device,
pretrained=pretrained_path)
self.tokenizer = {"name": "open_clip", "tokenizer": open_clip_lora.get_tokenizer(backbone)}
self.model = self.model.to(device)
self.model = self.model.float()
self.print_trainable_parameters()
def get_image_embeds(self,
image_embeds: torch.Tensor,
normalize: bool = True) -> torch.Tensor:
"""
Encode image_embeds according to the model below
:param image_embeds:
:param normalize:
:return:
"""
image_embeds = self.model.encode_image(image_embeds, normalize=normalize)
return image_embeds