From d3347910b7048a8b41c7d3c11093fbf34ef9efe1 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 19 Jul 2019 22:58:23 +0800 Subject: [PATCH] style: reformat code and optimize import --- gnes/encoder/base.py | 1 - gnes/encoder/image/base.py | 4 +- gnes/encoder/image/cvae.py | 8 ++- gnes/encoder/image/cvae_cores/model.py | 88 +++++++++++------------ gnes/encoder/image/inception.py | 7 +- gnes/encoder/numeric/tf_pq.py | 1 - gnes/encoder/text/elmo.py | 1 - gnes/encoder/text/gpt.py | 1 - gnes/preprocessor/helper.py | 6 +- gnes/preprocessor/image/base.py | 5 +- gnes/preprocessor/image/segmentation.py | 10 +-- gnes/preprocessor/image/sliding_window.py | 4 +- gnes/preprocessor/video/ffmpeg.py | 3 +- gnes/preprocessor/video/shotdetect.py | 5 +- gnes/router/base.py | 2 +- gnes/service/encoder.py | 2 +- gnes/service/grpc.py | 4 +- gnes/service/indexer.py | 2 +- gnes/service/preprocessor.py | 2 +- 19 files changed, 81 insertions(+), 75 deletions(-) diff --git a/gnes/encoder/base.py b/gnes/encoder/base.py index 05deae3b..bf3c1f77 100644 --- a/gnes/encoder/base.py +++ b/gnes/encoder/base.py @@ -21,7 +21,6 @@ import numpy as np from ..base import TrainableBase -from ..proto import gnes_pb2 class BaseEncoder(TrainableBase): diff --git a/gnes/encoder/image/base.py b/gnes/encoder/image/base.py index 90691dbd..6a92c108 100644 --- a/gnes/encoder/image/base.py +++ b/gnes/encoder/image/base.py @@ -53,12 +53,13 @@ def fn_parser(self, layer: str) -> Callable: if '(' not in layer and ')' not in layer: # this is a shorthand syntax we need to add "(x)" at the end - layer = 'm.%s(x)'%layer + layer = 'm.%s(x)' % layer else: pass def layer_fn(x, l, m, torch): return eval(l) + return lambda x: layer_fn(x, layer, self.m, torch) def forward(self, x): @@ -93,4 +94,3 @@ def encode(self, img: List['np.ndarray'], *args, **kwargs) -> np.ndarray: result_npy.append(encodes.data.cpu().numpy()) return np.array(result_npy, dtype=np.float32) - diff --git a/gnes/encoder/image/cvae.py b/gnes/encoder/image/cvae.py index e3b0e711..f297e17d 100644 --- a/gnes/encoder/image/cvae.py +++ b/gnes/encoder/image/cvae.py @@ -14,11 +14,13 @@ # limitations under the License. from typing import List + import numpy as np -from gnes.helper import batch_iterator -from ..base import BaseImageEncoder from PIL import Image +from ..base import BaseImageEncoder +from ...helper import batch_iterator + class CVAEEncoder(BaseImageEncoder): @@ -56,7 +58,7 @@ def post_init(self): def encode(self, img: List['np.ndarray'], *args, **kwargs) -> np.ndarray: ret = [] img = [(np.array(Image.fromarray(im).resize((120, 120)), - dtype=np.float32)/255) for im in img] + dtype=np.float32) / 255) for im in img] for _im in batch_iterator(img, self.batch_size): _mean, _var = self.sess.run((self.mean, self.var), feed_dict={self.inputs: _im}) diff --git a/gnes/encoder/image/cvae_cores/model.py b/gnes/encoder/image/cvae_cores/model.py index d928473d..9a348fac 100644 --- a/gnes/encoder/image/cvae_cores/model.py +++ b/gnes/encoder/image/cvae_cores/model.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import tensorflow as tf import numpy as np +import tensorflow as tf class CVAE(tf.keras.Model): @@ -23,53 +23,53 @@ def __init__(self, latent_dim): self.latent_dim = latent_dim self.inference_net = tf.keras.Sequential( [ - tf.keras.layers.InputLayer(input_shape=(120, 120, 3)), - tf.keras.layers.Conv2D( - filters=32, kernel_size=3, strides=(2, 2), - padding='SAME', - activation='relu'), - tf.keras.layers.Conv2D( - filters=32, kernel_size=3, strides=(2, 2), - padding='SAME', - activation='relu'), - tf.keras.layers.Conv2D( - filters=32, kernel_size=3, strides=(2, 2), - padding='SAME', - activation='relu'), - tf.keras.layers.Flatten(), - # No activation - tf.keras.layers.Dense(latent_dim + latent_dim), + tf.keras.layers.InputLayer(input_shape=(120, 120, 3)), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), + padding='SAME', + activation='relu'), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), + padding='SAME', + activation='relu'), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), + padding='SAME', + activation='relu'), + tf.keras.layers.Flatten(), + # No activation + tf.keras.layers.Dense(latent_dim + latent_dim), ] ) self.generative_net = tf.keras.Sequential( - [ - tf.keras.layers.InputLayer(input_shape=(latent_dim,)), - tf.keras.layers.Dense(units=15*15*32, - activation=tf.nn.relu), - tf.keras.layers.Reshape(target_shape=(15, 15, 32)), - tf.keras.layers.Conv2DTranspose( - filters=32, - kernel_size=3, - strides=(2, 2), - padding="SAME", - activation='relu'), - tf.keras.layers.Conv2DTranspose( - filters=32, - kernel_size=3, - strides=(2, 2), - padding="SAME", - activation='relu'), - tf.keras.layers.Conv2DTranspose( - filters=32, - kernel_size=3, - strides=(2, 2), - padding="SAME", - activation='relu'), - # No activation - tf.keras.layers.Conv2DTranspose( - filters=3, kernel_size=3, strides=(1, 1), padding="SAME"), - ] + [ + tf.keras.layers.InputLayer(input_shape=(latent_dim,)), + tf.keras.layers.Dense(units=15 * 15 * 32, + activation=tf.nn.relu), + tf.keras.layers.Reshape(target_shape=(15, 15, 32)), + tf.keras.layers.Conv2DTranspose( + filters=32, + kernel_size=3, + strides=(2, 2), + padding="SAME", + activation='relu'), + tf.keras.layers.Conv2DTranspose( + filters=32, + kernel_size=3, + strides=(2, 2), + padding="SAME", + activation='relu'), + tf.keras.layers.Conv2DTranspose( + filters=32, + kernel_size=3, + strides=(2, 2), + padding="SAME", + activation='relu'), + # No activation + tf.keras.layers.Conv2DTranspose( + filters=3, kernel_size=3, strides=(1, 1), padding="SAME"), + ] ) def sample(self, eps=None): diff --git a/gnes/encoder/image/inception.py b/gnes/encoder/image/inception.py index af143201..5c962146 100644 --- a/gnes/encoder/image/inception.py +++ b/gnes/encoder/image/inception.py @@ -14,10 +14,12 @@ # limitations under the License. from typing import List + import numpy as np +from PIL import Image + from ..base import BaseImageEncoder from ...helper import batching, batch_iterator -from PIL import Image class TFInceptionEncoder(BaseImageEncoder): @@ -63,7 +65,8 @@ def post_init(self): def encode(self, img: List['np.ndarray'], *args, **kwargs) -> np.ndarray: ret = [] img = [(np.array(Image.fromarray(im).resize((self.inception_size_x, - self.inception_size_y)), dtype=np.float32) * 2 / 255. - 1.) for im in img] + self.inception_size_y)), dtype=np.float32) * 2 / 255. - 1.) for im + in img] for _im in batch_iterator(img, self.batch_size): _, end_points_ = self.sess.run((self.logits, self.end_points), feed_dict={self.inputs: _im}) diff --git a/gnes/encoder/numeric/tf_pq.py b/gnes/encoder/numeric/tf_pq.py index eac20a43..1d051f11 100644 --- a/gnes/encoder/numeric/tf_pq.py +++ b/gnes/encoder/numeric/tf_pq.py @@ -70,6 +70,5 @@ def encode(self, vecs: np.ndarray, *args, **kwargs) -> np.ndarray: self._graph['ph_centroids']: self.centroids}) return tmp.astype(np.uint8) - def close(self): self._sess.close() diff --git a/gnes/encoder/text/elmo.py b/gnes/encoder/text/elmo.py index 10bfcbf9..64352db3 100644 --- a/gnes/encoder/text/elmo.py +++ b/gnes/encoder/text/elmo.py @@ -66,4 +66,3 @@ def encode(self, text: List[str], *args, **kwargs) -> np.ndarray: _pooled = pooling_np(_layer_data, self.pooling_strategy) pooled_data.append(_pooled) return np.array(pooled_data, dtype=np.float32) - diff --git a/gnes/encoder/text/gpt.py b/gnes/encoder/text/gpt.py index e5f64c1c..b7549aca 100644 --- a/gnes/encoder/text/gpt.py +++ b/gnes/encoder/text/gpt.py @@ -109,7 +109,6 @@ def encode(self, text: List[str], *args, **kwargs) -> np.ndarray: return output_tensor.numpy() - class GPT2Encoder(GPTEncoder): def _get_token_ids(self, x): diff --git a/gnes/preprocessor/helper.py b/gnes/preprocessor/helper.py index aa0a747d..31117eae 100644 --- a/gnes/preprocessor/helper.py +++ b/gnes/preprocessor/helper.py @@ -20,9 +20,9 @@ from typing import List, Callable import cv2 +import imagehash import numpy as np from PIL import Image -import imagehash def get_video_frames(buffer_data: bytes, image_format: str = "cv2", @@ -73,7 +73,7 @@ def get_video_frames(buffer_data: bytes, image_format: str = "cv2", def block_descriptor(image: "np.ndarray", descriptor_fn: Callable, num_blocks: int = 3) -> "np.ndarray": - h, w, _ = image.shape # find shape of image and channel + h, w, _ = image.shape # find shape of image and channel block_h = int(np.ceil(h / num_blocks)) block_w = int(np.ceil(w / num_blocks)) @@ -91,7 +91,7 @@ def pyramid_descriptor(image: "np.ndarray", max_level: int = 2) -> "np.ndarray": descriptors = [] for level in range(max_level + 1): - num_blocks = 2**level + num_blocks = 2 ** level descriptors.extend(block_descriptor(image, descriptor_fn, num_blocks)) return np.array(descriptors) diff --git a/gnes/preprocessor/image/base.py b/gnes/preprocessor/image/base.py index f92dfb0a..e2e72d56 100644 --- a/gnes/preprocessor/image/base.py +++ b/gnes/preprocessor/image/base.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + from ..base import BasePreprocessor from ...proto import gnes_pb2 -from typing import List class BaseImagePreprocessor(BasePreprocessor): @@ -35,4 +36,4 @@ def _get_all_chunks_weight(self, image_set: List['np.ndarray']) -> List[float]: def _torch_transform(cls, image): import torchvision.transforms as transforms return transforms.Compose([transforms.ToTensor(), - transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])(image) \ No newline at end of file + transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])(image) diff --git a/gnes/preprocessor/image/segmentation.py b/gnes/preprocessor/image/segmentation.py index 0dadbc46..ed31688a 100644 --- a/gnes/preprocessor/image/segmentation.py +++ b/gnes/preprocessor/image/segmentation.py @@ -1,10 +1,12 @@ -from .base import BaseImagePreprocessor -from ...proto import array2blob -from PIL import Image -import numpy as np import io import os +import numpy as np +from PIL import Image + +from .base import BaseImagePreprocessor +from ...proto import array2blob + class SegmentPreprocessor(BaseImagePreprocessor): diff --git a/gnes/preprocessor/image/sliding_window.py b/gnes/preprocessor/image/sliding_window.py index 4cccdd76..a0500143 100644 --- a/gnes/preprocessor/image/sliding_window.py +++ b/gnes/preprocessor/image/sliding_window.py @@ -81,7 +81,8 @@ def _get_all_sliding_window(self, img: 'np.ndarray') -> List['np.ndarray']: writeable=False ) expanded_input = expanded_input.reshape((-1, self.window_size, self.window_size, 3)) - return [np.array(Image.fromarray(img).resize((self.target_img_size, self.target_img_size))) for img in expanded_input] + return [np.array(Image.fromarray(img).resize((self.target_img_size, self.target_img_size))) for img in + expanded_input] class VanillaSlidingPreprocessor(BaseSlidingPreprocessor): @@ -94,4 +95,3 @@ class WeightedSlidingPreprocessor(BaseSlidingPreprocessor): def _get_all_chunks_weight(self, image_set: List['np.ndarray']) -> List[float]: return FFmpegPreprocessor.pic_weight(image_set) - diff --git a/gnes/preprocessor/video/ffmpeg.py b/gnes/preprocessor/video/ffmpeg.py index 33d6c488..80a1ab5a 100644 --- a/gnes/preprocessor/video/ffmpeg.py +++ b/gnes/preprocessor/video/ffmpeg.py @@ -14,11 +14,12 @@ # limitations under the License. from typing import List + import numpy as np from .base import BaseVideoPreprocessor -from ...proto import gnes_pb2, array2blob from ..helper import get_video_frames, phash_descriptor +from ...proto import gnes_pb2, array2blob class FFmpegPreprocessor(BaseVideoPreprocessor): diff --git a/gnes/preprocessor/video/shotdetect.py b/gnes/preprocessor/video/shotdetect.py index 29a86f58..377d8ee5 100644 --- a/gnes/preprocessor/video/shotdetect.py +++ b/gnes/preprocessor/video/shotdetect.py @@ -16,9 +16,10 @@ # pylint: disable=low-comment-ratio import numpy as np + from .base import BaseVideoPreprocessor -from ...proto import gnes_pb2, array2blob from ..helper import get_video_frames, compute_descriptor, compare_descriptor +from ...proto import gnes_pb2, array2blob class ShotDetectPreprocessor(BaseVideoPreprocessor): @@ -66,7 +67,7 @@ def apply(self, doc: 'gnes_pb2.Document') -> None: clt = KMeans(n_clusters=2) clt.fit(dists) - #select which cluster includes shot frames + # select which cluster includes shot frames big_center = np.argmax(clt.cluster_centers_) shots = [] diff --git a/gnes/router/base.py b/gnes/router/base.py index bd1af099..547a4cb2 100644 --- a/gnes/router/base.py +++ b/gnes/router/base.py @@ -14,7 +14,7 @@ # limitations under the License. # pylint: disable=low-comment-ratio -from typing import List, Optional, Generator +from typing import List, Generator from ..base import TrainableBase from ..proto import gnes_pb2, merge_routes diff --git a/gnes/service/encoder.py b/gnes/service/encoder.py index 9afb4d68..168c42f8 100644 --- a/gnes/service/encoder.py +++ b/gnes/service/encoder.py @@ -16,7 +16,7 @@ # pylint: disable=low-comment-ratio from typing import List, Union -from .base import BaseService as BS, MessageHandler, BlockMessage +from .base import BaseService as BS, MessageHandler from ..proto import gnes_pb2, array2blob, blob2array diff --git a/gnes/service/grpc.py b/gnes/service/grpc.py index b508eedb..1c881367 100644 --- a/gnes/service/grpc.py +++ b/gnes/service/grpc.py @@ -135,8 +135,8 @@ def __init__(self, args): self.logger = set_logger(self.__class__.__name__, args.verbose) self.server = grpc.server( futures.ThreadPoolExecutor(max_workers=args.max_concurrency), - options=[('grpc.max_send_message_length', args.max_send_size*1024*1024), - ('grpc.max_receive_message_length', args.max_receive_size*1024*1024)]) + options=[('grpc.max_send_message_length', args.max_send_size * 1024 * 1024), + ('grpc.max_receive_message_length', args.max_receive_size * 1024 * 1024)]) self.logger.info('start a grpc server with %d workers' % args.max_concurrency) gnes_pb2_grpc.add_GnesRPCServicer_to_server(GNESServicer(args), self.server) diff --git a/gnes/service/indexer.py b/gnes/service/indexer.py index 928cf960..6f6190c8 100644 --- a/gnes/service/indexer.py +++ b/gnes/service/indexer.py @@ -17,7 +17,7 @@ import numpy as np -from .base import BaseService as BS, ComponentNotLoad, MessageHandler, ServiceError +from .base import BaseService as BS, MessageHandler, ServiceError from ..proto import gnes_pb2, blob2array diff --git a/gnes/service/preprocessor.py b/gnes/service/preprocessor.py index fb98c294..15488667 100644 --- a/gnes/service/preprocessor.py +++ b/gnes/service/preprocessor.py @@ -15,7 +15,7 @@ # pylint: disable=low-comment-ratio -from .base import BaseService as BS, MessageHandler, ComponentNotLoad +from .base import BaseService as BS, MessageHandler from ..proto import gnes_pb2