From f8e18d067722a9454e13a978cf7be41ea7241ed3 Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Mon, 9 Sep 2019 12:24:44 +0800 Subject: [PATCH 1/5] fix(encoder): fix vald in numeric encoder --- tests/test_vlad.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/test_vlad.py diff --git a/tests/test_vlad.py b/tests/test_vlad.py new file mode 100644 index 00000000..732f8db2 --- /dev/null +++ b/tests/test_vlad.py @@ -0,0 +1,26 @@ +import os +import unittest +import numpy as np +from gnes.encoder.numeric.vlad import VladEncoder + + +class TestVladEncoder(unittest.TestCase): + def setUp(self): + self.mock_train_data = np.random.random([200, 128]) + self.mock_eval_data = np.random.random([2, 2, 128]) + self.dump_path = os.path.join(os.path.dirname(__file__), 'vlad.bin') + + def tearDown(self): + if os.path.exists(self.dump_path): + os.remove(self.dump_path) + + def test_vlad_train(self): + model = VladEncoder(20) + model.train(self.mock_train_data) + self.assertEqual(model.centroids.shape, (20, 128)) + model.dump(self.dump_path) + + def test_vlad_encode(self): + model = VladEncoder.load(self.dump_path) + v = model.encode(self.mock_eval_data) + self.assertEqual(v.shape, (2, 2560)) From 1ba4e11cb7f18b97cb35faed61b7d82fb512cd84 Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Mon, 9 Sep 2019 12:25:20 +0800 Subject: [PATCH 2/5] fix(encoder): fix vald encoder and add unittest --- gnes/encoder/numeric/vlad.py | 40 ++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/gnes/encoder/numeric/vlad.py b/gnes/encoder/numeric/vlad.py index 39664aeb..9a6cf63c 100644 --- a/gnes/encoder/numeric/vlad.py +++ b/gnes/encoder/numeric/vlad.py @@ -25,22 +25,31 @@ class VladEncoder(BaseNumericEncoder): batch_size = 2048 - def __init__(self, num_clusters: int, *args, **kwargs): + def __init__(self, num_clusters: int, + using_faiss_pred: True, + *args, **kwargs): super().__init__(*args, **kwargs) self.num_clusters = num_clusters + self.using_faiss_pred = using_faiss_pred self.centroids = None + self.index_flat = None def kmeans_train(self, vecs): import faiss - kmeans = faiss.Kmeans(vecs.shape[1], self.num_clusters, niter=5, verbose=False) kmeans.train(vecs) self.centroids = kmeans.centroids + self.index_flat = faiss.IndexFlatL2(self.centroids.shape[1]) + self.index_flat.add(self.centroids) def kmeans_pred(self, vecs): - vecs = np.reshape(vecs, [vecs.shape[0], 1, 1, vecs.shape[1]]) - dist = np.sum(np.square(vecs - self.centroids), -1) - return np.argmax(-dist, axis=-1).astype(np.int64) + if self.using_faiss_pred: + D, I = self.index_flat.search(vecs.astype(np.float32), 1) + return np.reshape(I, [-1]) + else: + vecs = np.reshape(vecs, [vecs.shape[0], 1, 1, vecs.shape[1]]) + dist = np.sum(np.square(vecs - self.centroids), -1) + return np.argmax(-dist, axis=-1).reshape([-1]).astype(np.int32) @batching def train(self, vecs: np.ndarray, *args, **kwargs): @@ -52,24 +61,25 @@ def train(self, vecs: np.ndarray, *args, **kwargs): @train_required @batching def encode(self, vecs: np.ndarray, *args, **kwargs) -> np.ndarray: - vecs_ = copy.deepcopy(vecs) - vecs_ = np.concatenate((list(vecs_[i] for i in range(len(vecs_)))), axis=0) - - knn_output = self.kmeans_pred(vecs_) - knn_output = [knn_output[i:i + vecs.shape[1]] for i in range(0, len(knn_output), vecs.shape[1])] + knn_output = [self.kmeans_pred(vecs_) for vecs_ in vecs] output = [] for chunk_count, chunk in enumerate(vecs): res = np.zeros((self.centroids.shape[0], self.centroids.shape[1])) for frame_count, frame in enumerate(chunk): - center_index = knn_output[chunk_count][frame_count][0] + center_index = knn_output[chunk_count][frame_count] res[center_index] += (frame - self.centroids[center_index]) - output.append(res) + res = res.reshape([-1]) + output.append(res / np.sum(res**2)**0.5) - output = np.array(list(map(lambda x: x.reshape(1, -1), output)), dtype=np.float32) - output = np.squeeze(output, axis=1) - return output + return np.array(output, dtype=np.float32) def _copy_from(self, x: 'VladEncoder') -> None: self.num_clusters = x.num_clusters self.centroids = x.centroids + self.using_faiss_pred = x.using_faiss_pred + if self.using_faiss_pred: + import faiss + self.index_flat = faiss.IndexFlatL2(self.centroids.shape[1]) + self.index_flat.add(self.centroids) + From ddf13ff1ecf014599a32031e2071f0b57336fe55 Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Mon, 9 Sep 2019 12:33:41 +0800 Subject: [PATCH 3/5] fix(encoder): fix bug in vlad encoder --- gnes/encoder/numeric/vlad.py | 2 +- tests/test_vlad.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/gnes/encoder/numeric/vlad.py b/gnes/encoder/numeric/vlad.py index 9a6cf63c..d64aad5b 100644 --- a/gnes/encoder/numeric/vlad.py +++ b/gnes/encoder/numeric/vlad.py @@ -26,7 +26,7 @@ class VladEncoder(BaseNumericEncoder): batch_size = 2048 def __init__(self, num_clusters: int, - using_faiss_pred: True, + using_faiss_pred: bool=True, *args, **kwargs): super().__init__(*args, **kwargs) self.num_clusters = num_clusters diff --git a/tests/test_vlad.py b/tests/test_vlad.py index 732f8db2..814ea866 100644 --- a/tests/test_vlad.py +++ b/tests/test_vlad.py @@ -18,9 +18,12 @@ def test_vlad_train(self): model = VladEncoder(20) model.train(self.mock_train_data) self.assertEqual(model.centroids.shape, (20, 128)) - model.dump(self.dump_path) - - def test_vlad_encode(self): - model = VladEncoder.load(self.dump_path) v = model.encode(self.mock_eval_data) self.assertEqual(v.shape, (2, 2560)) + + def test_vlad_dump_load(self): + model = VladEncoder(20) + model.train(self.mock_train_data) + model.dump(self.dump_path) + model_new = VladEncoder.load(self.dump_path) + self.assertEqual(model_new.centroids.shape, (20, 128)) From ffc822b39e39dad05fbfa84f28a4844a31d3e785 Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Mon, 9 Sep 2019 13:07:55 +0800 Subject: [PATCH 4/5] fix(encoder): fix vlad unittest --- gnes/encoder/numeric/vlad.py | 25 +++++++++++++++++-------- tests/test_vlad.py | 4 ++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/gnes/encoder/numeric/vlad.py b/gnes/encoder/numeric/vlad.py index d64aad5b..d8d1cb0b 100644 --- a/gnes/encoder/numeric/vlad.py +++ b/gnes/encoder/numeric/vlad.py @@ -14,8 +14,6 @@ # limitations under the License. -import copy - import numpy as np from ..base import BaseNumericEncoder @@ -39,6 +37,11 @@ def kmeans_train(self, vecs): kmeans = faiss.Kmeans(vecs.shape[1], self.num_clusters, niter=5, verbose=False) kmeans.train(vecs) self.centroids = kmeans.centroids + if self.using_faiss_pred: + self.faiss_index() + + def faiss_index(self): + import faiss self.index_flat = faiss.IndexFlatL2(self.centroids.shape[1]) self.index_flat.add(self.centroids) @@ -53,10 +56,9 @@ def kmeans_pred(self, vecs): @batching def train(self, vecs: np.ndarray, *args, **kwargs): + vecs = vecs.reshape([-1, vecs.shape[-1]]) assert len(vecs) > self.num_clusters, 'number of data should be larger than number of clusters' - vecs_ = copy.deepcopy(vecs) - vecs_ = np.concatenate((list(vecs_[i] for i in range(len(vecs_)))), axis=0) - self.kmeans_train(vecs_) + self.kmeans_train(vecs) @train_required @batching @@ -79,7 +81,14 @@ def _copy_from(self, x: 'VladEncoder') -> None: self.centroids = x.centroids self.using_faiss_pred = x.using_faiss_pred if self.using_faiss_pred: - import faiss - self.index_flat = faiss.IndexFlatL2(self.centroids.shape[1]) - self.index_flat.add(self.centroids) + self.faiss_index() + + def __setstate__(self, state): + super().__setstate__(state) + if self.using_faiss_pred: + self.faiss_index() + def __getstate__(self): + state = super().__getstate__() + del state['index_flat'] + return state diff --git a/tests/test_vlad.py b/tests/test_vlad.py index 814ea866..66132e6d 100644 --- a/tests/test_vlad.py +++ b/tests/test_vlad.py @@ -6,8 +6,8 @@ class TestVladEncoder(unittest.TestCase): def setUp(self): - self.mock_train_data = np.random.random([200, 128]) - self.mock_eval_data = np.random.random([2, 2, 128]) + self.mock_train_data = np.random.random([1, 200, 128]).astype(np.float32) + self.mock_eval_data = np.random.random([2, 2, 128]).astype(np.float32) self.dump_path = os.path.join(os.path.dirname(__file__), 'vlad.bin') def tearDown(self): From 814b2ee6a8d2cf8c3b436c02c15176ca227233dc Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Mon, 9 Sep 2019 13:52:21 +0800 Subject: [PATCH 5/5] fix(encoder): fix vald encocer --- gnes/encoder/numeric/vlad.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnes/encoder/numeric/vlad.py b/gnes/encoder/numeric/vlad.py index d8d1cb0b..433a0c97 100644 --- a/gnes/encoder/numeric/vlad.py +++ b/gnes/encoder/numeric/vlad.py @@ -47,8 +47,8 @@ def faiss_index(self): def kmeans_pred(self, vecs): if self.using_faiss_pred: - D, I = self.index_flat.search(vecs.astype(np.float32), 1) - return np.reshape(I, [-1]) + _, pred = self.index_flat.search(vecs.astype(np.float32), 1) + return np.reshape(pred, [-1]) else: vecs = np.reshape(vecs, [vecs.shape[0], 1, 1, vecs.shape[1]]) dist = np.sum(np.square(vecs - self.centroids), -1)