-
Notifications
You must be signed in to change notification settings - Fork 19
/
Copy pathmodel.py
54 lines (44 loc) · 1.61 KB
/
model.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from __future__ import annotations
from typing import TYPE_CHECKING
import numpy as np
from gensim.models import KeyedVectors
if TYPE_CHECKING:
from typing import AsyncIterator
from common.typing import np_float_arr
class GensimModel:
def __init__(self, model: KeyedVectors):
self.model = model
async def get_vector(self, word: str) -> np_float_arr | None:
if not all(ord("א") <= ord(c) <= ord("ת") for c in word):
return None
if len(word) == 1:
return None
if word not in self.model:
return None
vector: np_float_arr = self.model[word].tolist()
return vector
async def get_similarities(
self, words: list[str], vector: np_float_arr
) -> np_float_arr:
similarities: np_float_arr = np.round(
self.model.cosine_similarities(
vector, np.asarray([self.model[w] for w in words])
)
* 100,
2,
)
return similarities
async def iterate_all(self) -> AsyncIterator[tuple[str, np_float_arr]]:
for word in self.model.key_to_index.keys():
if isinstance(word, str):
vector = await self.get_vector(word)
else:
continue
if vector is None:
continue
yield word, self.model[word]
async def calc_similarity(self, vec1: np_float_arr, vec2: np_float_arr) -> float:
similarities: np_float_arr = self.model.cosine_similarities(
vec1, np.expand_dims(vec2, axis=0)
)
return round(float(similarities[0]) * 100, 2)