diff --git a/.gitignore b/.gitignore index 822f94d..a02b768 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ video_tmp/ result/ nohup.out #./ +/.vscode /pix2pix /pix2pixHD /tmp @@ -183,4 +184,5 @@ nohup.out *.MP4 *.JPEG *.exe -*.npy \ No newline at end of file +*.npy +*.psd \ No newline at end of file diff --git a/README.md b/README.md index 21dd96f..9ebb067 100755 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ -![image](./imgs/hand.gif) -# DeepMosaics -You can use it to automatically remove the mosaics in images and videos, or add mosaics to them.
-This project is based on "semantic segmentation" and "Image-to-Image Translation".
- -* [中文版README](./README_CN.md)
+
+

+      +
-### More Examples +# DeepMosaics +**English | [中文](./README_CN.md)**
+You can use it to automatically remove the mosaics in images and videos, or add mosaics to them.
This project is based on "semantic segmentation" and "Image-to-Image Translation".
Try it at this [website](http://118.89.27.46:5000/)!
+### Examples +![image](./imgs/hand.gif) origin | auto add mosaic | auto clean mosaic :-:|:-:|:-: ![image](./imgs/example/lena.jpg) | ![image](./imgs/example/lena_add.jpg) | ![image](./imgs/example/lena_clean.jpg) @@ -30,18 +32,21 @@ An interesting example:[Ricardo Milos to cat](https://www.bilibili.com/video/BV1 ## Run DeepMosaics You can either run DeepMosaics via a pre-built binary package, or from source.
+### Try it on web +You can simply try to remove the mosaic on the face at this [website](http://118.89.27.46:5000/).
### Pre-built binary package For Windows, we bulid a GUI version for easy testing.
Download this version, and a pre-trained model via [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs) [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ)
-* [[How to use]](./docs/exe_help.md)
+* [[Help document]](./docs/exe_help.md)
+* Video tutorial => [[youtube]](https://www.youtube.com/watch?v=1kEmYawJ_vk) [[bilibili]](https://www.bilibili.com/video/BV1QK4y1a7Av) ![image](./imgs/GUI.png)
Attentions:
- Requires Windows_x86_64, Windows10 is better.
- Different pre-trained models are suitable for different effects.[[Introduction to pre-trained models]](./docs/pre-trained_models_introduction.md)
- - Run time depends on computers performance(The current version does not support gpu, if you need to use gpu please run source).
+ - Run time depends on computers performance (GPU version has better performance but requires CUDA to be installed).
- If output video cannot be played, you can try with [potplayer](https://daumpotplayer.com/download/).
- GUI version updates slower than source.
@@ -67,11 +72,11 @@ You can download pre_trained models and put them into './pretrained_models'.
#### Simple Example * Add Mosaic (output media will save in './result')
```bash -python deepmosaic.py --media_path ./imgs/ruoruo.jpg --model_path ./pretrained_models/mosaic/add_face.pth --use_gpu 0 +python deepmosaic.py --media_path ./imgs/ruoruo.jpg --model_path ./pretrained_models/mosaic/add_face.pth --gpu_id 0 ``` * Clean Mosaic (output media will save in './result')
```bash -python deepmosaic.py --media_path ./result/ruoruo_add.jpg --model_path ./pretrained_models/mosaic/clean_face_HD.pth --use_gpu 0 +python deepmosaic.py --media_path ./result/ruoruo_add.jpg --model_path ./pretrained_models/mosaic/clean_face_HD.pth --gpu_id 0 ``` #### More Parameters If you want to test other images or videos, please refer to this file.
@@ -81,5 +86,4 @@ If you want to test other images or videos, please refer to this file.
If you want to train with your own dataset, please refer to [training_with_your_own_dataset.md](./docs/training_with_your_own_dataset.md) ## Acknowledgements -This code borrows heavily from [[pytorch-CycleGAN-and-pix2pix]](https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix) [[Pytorch-UNet]](https://github.com/milesial/Pytorch-UNet) [[pix2pixHD]](https://github.com/NVIDIA/pix2pixHD) [[BiSeNet]](https://github.com/ooooverflow/BiSeNet). - +This code borrows heavily from [[pytorch-CycleGAN-and-pix2pix]](https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix) [[Pytorch-UNet]](https://github.com/milesial/Pytorch-UNet) [[pix2pixHD]](https://github.com/NVIDIA/pix2pixHD) [[BiSeNet]](https://github.com/ooooverflow/BiSeNet) [[DFDNet]](https://github.com/csxmli2016/DFDNet) [[GFRNet_pytorch_new]](https://github.com/sonack/GFRNet_pytorch_new). diff --git a/README_CN.md b/README_CN.md index de37161..090371d 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,9 +1,15 @@ -![image](./imgs/hand.gif) -# DeepMosaics -这是一个通过深度学习自动的为图片/视频添加马赛克,或消除马赛克的项目.
它基于“语义分割”以及“图像翻译”.
+
+

+      +
+ +# DeepMosaics +**[English](./README.md) | 中文**
-### 更多例子 +这是一个通过深度学习自动的为图片/视频添加马赛克,或消除马赛克的项目.
它基于“语义分割”以及“图像翻译”.
现在可以在这个[网站](http://118.89.27.46:5000/)尝试使用该项目清除马赛克!
+### 例子 +![image](./imgs/hand.gif) 原始 | 自动打码 | 自动去码 :-:|:-:|:-: ![image](./imgs/example/lena.jpg) | ![image](./imgs/example/lena_add.jpg) | ![image](./imgs/example/lena_clean.jpg) @@ -26,19 +32,20 @@ ## 如何运行 可以通过我们预编译好的二进制包或源代码运行.
- +### 在网页中运行 +打开[这个网站](http://118.89.27.46:5000/)上传照片，将获得去除马赛克后的结果，受限与当地法律，目前只支持人脸.
### 预编译的程序包 对于Windows用户,我们提供了包含GUI界面的免安装软件包.
可以通过下面两种方式进行下载: [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs) [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ)
-* [[使用教程]](./docs/exe_help_CN.md)
- +* [[帮助文档]](./docs/exe_help_CN.md)
+* [[视频教程]](https://www.bilibili.com/video/BV1QK4y1a7Av)
![image](./imgs/GUI.png)
注意事项:
- 程序的运行要求在64位Windows操作系统,我仅在Windows10运行过,其他版本暂未经过测试
- 请根据需求选择合适的预训练模型进行测试，不同的预期训练模型具有不同的效果.[[预训练模型介绍]](./docs/pre-trained_models_introduction_CN.md)
- - 运行时间取决于电脑性能,对于视频文件,我们建议使用源码并在GPU上运行.
+ - 运行时间取决于电脑性能,对于视频文件,我们建议在GPU上运行.
- 如果输出的视频无法播放,这边建议您尝试[potplayer](https://daumpotplayer.com/download/).
- 相比于源码,该版本的更新将会延后. @@ -62,13 +69,13 @@ cd DeepMosaics [[预训练模型介绍]](./docs/pre-trained_models_introduction_CN.md)
#### 简单的例子 -* 为视频添加马赛克,例子中认为脸是需要打码的区域 ,可以通过切换预训练模型切换自动打码区域(输出结果将储存到 './result')
+* 为视频或照片添加马赛克,例子中认为脸是需要打码的区域 ,可以通过切换预训练模型切换自动打码区域(输出结果将储存到 './result')
```bash -python deepmosaic.py --media_path ./imgs/ruoruo.jpg --model_path ./pretrained_models/mosaic/add_face.pth --use_gpu 0 +python deepmosaic.py --media_path ./imgs/ruoruo.jpg --model_path ./pretrained_models/mosaic/add_face.pth --gpu_id 0 ``` -* 将视频中的马赛克移除,对于不同的打码物体需要使用对应的预训练模型进行马赛克消除(输出结果将储存到 './result')
+* 将视频或照片中的马赛克移除,对于不同的打码物体需要使用对应的预训练模型进行马赛克消除(输出结果将储存到 './result')
```bash -python deepmosaic.py --media_path ./result/ruoruo_add.jpg --model_path ./pretrained_models/mosaic/clean_face_HD.pth --use_gpu 0 +python deepmosaic.py --media_path ./result/ruoruo_add.jpg --model_path ./pretrained_models/mosaic/clean_face_HD.pth --gpu_id 0 ``` #### 更多的参数 如果想要测试其他的图片或视频,请参照以下文件输入参数.
@@ -78,5 +85,5 @@ python deepmosaic.py --media_path ./result/ruoruo_add.jpg --model_path ./pretrai 如果需要使用自己的数据训练模型，请参照 [training_with_your_own_dataset.md](./docs/training_with_your_own_dataset.md) ## 鸣谢 -代码大量的参考了以下项目:[[pytorch-CycleGAN-and-pix2pix]](https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix) [[Pytorch-UNet]](https://github.com/milesial/Pytorch-UNet) [[pix2pixHD]](https://github.com/NVIDIA/pix2pixHD) [[BiSeNet]](https://github.com/ooooverflow/BiSeNet). +代码大量的参考了以下项目:[[pytorch-CycleGAN-and-pix2pix]](https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix) [[Pytorch-UNet]](https://github.com/milesial/Pytorch-UNet) [[pix2pixHD]](https://github.com/NVIDIA/pix2pixHD) [[BiSeNet]](https://github.com/ooooverflow/BiSeNet) [[DFDNet]](https://github.com/csxmli2016/DFDNet) [[GFRNet_pytorch_new]](https://github.com/sonack/GFRNet_pytorch_new). diff --git a/cores/core.py b/cores/core.py index db99217..f9cc64c 100644 --- a/cores/core.py +++ b/cores/core.py @@ -1,5 +1,6 @@ import os import time +import torch import numpy as np import cv2 @@ -30,6 +31,7 @@ def video_init(opt,path): continue_flag = True if not continue_flag: + print('Step:1/4 -- Convert video to images') util.file_init(opt) ffmpeg.video2voice(path,opt.temp_dir+'/voice_tmp.mp3',opt.start_time,opt.last_time) ffmpeg.video2image(path,opt.temp_dir+'/video2image/output_%06d.'+opt.tempimage_type,fps,opt.start_time,opt.last_time) @@ -59,7 +61,7 @@ def addmosaic_video(opt,netS): if not opt.no_preview: cv2.namedWindow('preview', cv2.WINDOW_NORMAL) - print('Find ROI location:') + print('Step:2/4 -- Find ROI location') for i,imagepath in enumerate(imagepaths,1): img = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath)) mask,x,y,size,area = runmodel.get_ROI_position(img,netS,opt) @@ -77,7 +79,7 @@ def addmosaic_video(opt,netS): mask_index = filt.position_medfilt(np.array(positions), 7) # add mosaic - print('Add Mosaic:') + print('Step:3/4 -- Add Mosaic:') t1 = time.time() for i,imagepath in enumerate(imagepaths,1): mask = impro.imread(os.path.join(opt.temp_dir+'/ROI_mask',imagepaths[mask_index[i-1]]),'gray') @@ -100,6 +102,7 @@ def addmosaic_video(opt,netS): print() if not opt.no_preview: cv2.destroyAllWindows() + print('Step:4/4 -- Convert images to video') ffmpeg.image2video( fps, opt.temp_dir+'/addmosaic_image/output_%06d.'+opt.tempimage_type, opt.temp_dir+'/voice_tmp.mp3', @@ -119,7 +122,7 @@ def styletransfer_video(opt,netG): path = opt.media_path positions = [] fps,imagepaths = video_init(opt,path)[:2] - print('Transfer:') + print('Step:2/4 -- Transfer') t1 = time.time() if not opt.no_preview: cv2.namedWindow('preview', cv2.WINDOW_NORMAL) @@ -142,6 +145,7 @@ def styletransfer_video(opt,netG): if not opt.no_preview: cv2.destroyAllWindows() suffix = os.path.basename(opt.model_path).replace('.pth','').replace('style_','') + print('Step:4/4 -- Convert images to video') ffmpeg.image2video( fps, opt.temp_dir+'/style_transfer/output_%06d.'+opt.tempimage_type, opt.temp_dir+'/voice_tmp.mp3', @@ -156,8 +160,7 @@ def get_mosaic_positions(opt,netM,imagepaths,savemask=True): t1 = time.time() if not opt.no_preview: cv2.namedWindow('mosaic mask', cv2.WINDOW_NORMAL) - - print('Find mosaic location:') + print('Step:2/4 -- Find mosaic location') for i,imagepath in enumerate(imagepaths,1): img_origin = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath)) x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt) @@ -186,7 +189,7 @@ def cleanmosaic_img(opt,netG,netM): print('Clean Mosaic:',path) img_origin = impro.imread(path) x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt) - cv2.imwrite('./mask/'+os.path.basename(path), mask) + #cv2.imwrite('./mask/'+os.path.basename(path), mask) img_result = img_origin.copy() if size > 100 : img_mosaic = img_origin[y-size:y+size,x-size:x+size] @@ -199,6 +202,18 @@ def cleanmosaic_img(opt,netG,netM): print('Do not find mosaic') impro.imwrite(os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.jpg'),img_result) +def cleanmosaic_img_server(opt,img_origin,netG,netM): + x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt) + img_result = img_origin.copy() + if size > 100 : + img_mosaic = img_origin[y-size:y+size,x-size:x+size] + if opt.traditional: + img_fake = runmodel.traditional_cleaner(img_mosaic,opt) + else: + img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt) + img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather) + return img_result + def cleanmosaic_video_byframe(opt,netG,netM): path = opt.media_path fps,imagepaths = video_init(opt,path)[:2] @@ -208,7 +223,7 @@ def cleanmosaic_video_byframe(opt,netG,netM): cv2.namedWindow('clean', cv2.WINDOW_NORMAL) # clean mosaic - print('Clean Mosaic:') + print('Step:3/4 -- Clean Mosaic:') length = len(imagepaths) for i,imagepath in enumerate(imagepaths,0): x,y,size = positions[i][0],positions[i][1],positions[i][2] @@ -237,6 +252,7 @@ def cleanmosaic_video_byframe(opt,netG,netM): print() if not opt.no_preview: cv2.destroyAllWindows() + print('Step:4/4 -- Convert images to video') ffmpeg.image2video( fps, opt.temp_dir+'/replace_mosaic/output_%06d.'+opt.tempimage_type, opt.temp_dir+'/voice_tmp.mp3', @@ -244,11 +260,15 @@ def cleanmosaic_video_byframe(opt,netG,netM): def cleanmosaic_video_fusion(opt,netG,netM): path = opt.media_path - N = 25 - if 'HD' in os.path.basename(opt.model_path): - INPUT_SIZE = 256 - else: - INPUT_SIZE = 128 + N,T,S = 2,5,3 + LEFT_FRAME = (N*S) + POOL_NUM = LEFT_FRAME*2+1 + INPUT_SIZE = 256 + FRAME_POS = np.linspace(0, (T-1)*S,T,dtype=np.int64) + img_pool = [] + previous_frame = None + init_flag = True + fps,imagepaths,height,width = video_init(opt,path) positions = get_mosaic_positions(opt,netM,imagepaths,savemask=True) t1 = time.time() @@ -256,39 +276,45 @@ def cleanmosaic_video_fusion(opt,netG,netM): cv2.namedWindow('clean', cv2.WINDOW_NORMAL) # clean mosaic - print('Clean Mosaic:') + print('Step:3/4 -- Clean Mosaic:') length = len(imagepaths) - - img_pool = [] - mosaic_input = np.zeros((INPUT_SIZE,INPUT_SIZE,3*N+1), dtype='uint8') - + for i,imagepath in enumerate(imagepaths,0): x,y,size = positions[i][0],positions[i][1],positions[i][2] - + input_stream = [] # image read stream - mask = cv2.imread(os.path.join(opt.temp_dir+'/mosaic_mask',imagepath),0) - if i==0 : - for j in range(0,N): - img_pool.append(impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepaths[np.clip(i+j-12,0,len(imagepaths)-1)]))) - else: + if i==0 :# init + for j in range(POOL_NUM): + img_pool.append(impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepaths[np.clip(i+j-LEFT_FRAME,0,len(imagepaths)-1)]))) + else: # load next frame img_pool.pop(0) - img_pool.append(impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepaths[np.clip(i+12,0,len(imagepaths)-1)]))) - img_origin = img_pool[12] + img_pool.append(impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepaths[np.clip(i+LEFT_FRAME,0,len(imagepaths)-1)]))) + img_origin = img_pool[LEFT_FRAME] img_result = img_origin.copy() - if size>100: + if size>50: try:#Avoid unknown errors - #reshape to network input shape - for k in range(N): - mosaic_input[:,:,k*3:(k+1)*3] = impro.resize(img_pool[k][y-size:y+size,x-size:x+size], INPUT_SIZE) - mask_input = impro.resize(mask,np.min(img_origin.shape[:2]))[y-size:y+size,x-size:x+size] - mosaic_input[:,:,-1] = impro.resize(mask_input, INPUT_SIZE) - mosaic_input_tensor = data.im2tensor(mosaic_input,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1 = False) - unmosaic_pred = netG(mosaic_input_tensor) - img_fake = data.tensor2im(unmosaic_pred,rgb2bgr = False ,is0_1 = False) + for pos in FRAME_POS: + input_stream.append(impro.resize(img_pool[pos][y-size:y+size,x-size:x+size], INPUT_SIZE)[:,:,::-1]) + if init_flag: + init_flag = False + previous_frame = input_stream[N] + previous_frame = data.im2tensor(previous_frame,bgr2rgb=True,gpu_id=opt.gpu_id) + + input_stream = np.array(input_stream).reshape(1,T,INPUT_SIZE,INPUT_SIZE,3).transpose((0,4,1,2,3)) + input_stream = data.to_tensor(data.normalize(input_stream),gpu_id=opt.gpu_id) + with torch.no_grad(): + unmosaic_pred = netG(input_stream,previous_frame) + img_fake = data.tensor2im(unmosaic_pred,rgb2bgr = True) + previous_frame = unmosaic_pred + # previous_frame = data.tensor2im(unmosaic_pred,rgb2bgr = True) + mask = cv2.imread(os.path.join(opt.temp_dir+'/mosaic_mask',imagepath),0) img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather) except Exception as e: - print('Warning:',e) + init_flag = True + print('Error:',e) + else: + init_flag = True cv2.imwrite(os.path.join(opt.temp_dir+'/replace_mosaic',imagepath),img_result) os.remove(os.path.join(opt.temp_dir+'/video2image',imagepath)) @@ -301,6 +327,7 @@ def cleanmosaic_video_fusion(opt,netG,netM): print() if not opt.no_preview: cv2.destroyAllWindows() + print('Step:4/4 -- Convert images to video') ffmpeg.image2video( fps, opt.temp_dir+'/replace_mosaic/output_%06d.'+opt.tempimage_type, opt.temp_dir+'/voice_tmp.mp3', diff --git a/cores/options.py b/cores/options.py index 7e8c165..bba8bae 100644 --- a/cores/options.py +++ b/cores/options.py @@ -11,7 +11,8 @@ def __init__(self): def initialize(self): #base - self.parser.add_argument('--use_gpu', type=int,default=0, help='if -1, use cpu') + self.parser.add_argument('--debug', action='store_true', help='if specified, start debug mode') + self.parser.add_argument('--gpu_id', type=str,default='0', help='if -1, use cpu') self.parser.add_argument('--media_path', type=str, default='./imgs/ruoruo.jpg',help='your videos or images path') self.parser.add_argument('-ss', '--start_time', type=str, default='00:00:00',help='start position of video, default is the beginning of video') self.parser.add_argument('-t', '--last_time', type=str, default='00:00:00',help='duration of the video, default is the entire video') @@ -59,62 +60,71 @@ def getparse(self, test_flag = False): model_name = os.path.basename(self.opt.model_path) self.opt.temp_dir = os.path.join(self.opt.temp_dir, 'DeepMosaics_temp') - os.environ["CUDA_VISIBLE_DEVICES"] = str(self.opt.use_gpu) - import torch - if torch.cuda.is_available() and self.opt.use_gpu > -1: - pass - else: - self.opt.use_gpu = -1 + if self.opt.gpu_id != '-1': + os.environ["CUDA_VISIBLE_DEVICES"] = str(self.opt.gpu_id) + import torch + if not torch.cuda.is_available(): + self.opt.gpu_id = '-1' + # else: + # self.opt.gpu_id = '-1' if test_flag: if not os.path.exists(self.opt.media_path): - print('Error: Bad media path!') + print('Error: Media does not exist!') input('Please press any key to exit.\n') sys.exit(0) - - if self.opt.mode == 'auto': - if 'clean' in model_name or self.opt.traditional: - self.opt.mode = 'clean' - elif 'add' in model_name: - self.opt.mode = 'add' - elif 'style' in model_name or 'edges' in model_name: - self.opt.mode = 'style' - else: - print('Please input running model!') - input('Please press any key to exit.\n') - sys.exit(0) - - if self.opt.output_size == 0 and self.opt.mode == 'style': - self.opt.output_size = 512 - - if 'edges' in model_name or 'edges' in self.opt.preprocess: - self.opt.edges = True - - if self.opt.netG == 'auto' and self.opt.mode =='clean': - if 'unet_128' in model_name: - self.opt.netG = 'unet_128' - elif 'resnet_9blocks' in model_name: - self.opt.netG = 'resnet_9blocks' - elif 'HD' in model_name and 'video' not in model_name: - self.opt.netG = 'HD' - elif 'video' in model_name: - self.opt.netG = 'video' - else: - print('Type of Generator error!') + if not os.path.exists(self.opt.model_path): + print('Error: Model does not exist!') input('Please press any key to exit.\n') sys.exit(0) - if self.opt.ex_mult == 'auto': - if 'face' in model_name: - self.opt.ex_mult = 1.1 + if self.opt.mode == 'auto': + if 'clean' in model_name or self.opt.traditional: + self.opt.mode = 'clean' + elif 'add' in model_name: + self.opt.mode = 'add' + elif 'style' in model_name or 'edges' in model_name: + self.opt.mode = 'style' + else: + print('Please check model_path!') + input('Please press any key to exit.\n') + sys.exit(0) + + if self.opt.output_size == 0 and self.opt.mode == 'style': + self.opt.output_size = 512 + + if 'edges' in model_name or 'edges' in self.opt.preprocess: + self.opt.edges = True + + if self.opt.netG == 'auto' and self.opt.mode =='clean': + if 'unet_128' in model_name: + self.opt.netG = 'unet_128' + elif 'resnet_9blocks' in model_name: + self.opt.netG = 'resnet_9blocks' + elif 'HD' in model_name and 'video' not in model_name: + self.opt.netG = 'HD' + elif 'video' in model_name: + self.opt.netG = 'video' + else: + print('Type of Generator error!') + input('Please press any key to exit.\n') + sys.exit(0) + + if self.opt.ex_mult == 'auto': + if 'face' in model_name: + self.opt.ex_mult = 1.1 + else: + self.opt.ex_mult = 1.5 else: - self.opt.ex_mult = 1.5 - else: - self.opt.ex_mult = float(self.opt.ex_mult) - - if self.opt.mosaic_position_model_path == 'auto': - _path = os.path.join(os.path.split(self.opt.model_path)[0],'mosaic_position.pth') - self.opt.mosaic_position_model_path = _path - # print(self.opt.mosaic_position_model_path) + self.opt.ex_mult = float(self.opt.ex_mult) + + if self.opt.mosaic_position_model_path == 'auto' and self.opt.mode == 'clean': + _path = os.path.join(os.path.split(self.opt.model_path)[0],'mosaic_position.pth') + if os.path.isfile(_path): + self.opt.mosaic_position_model_path = _path + else: + input('Please check mosaic_position_model_path!') + input('Please press any key to exit.\n') + sys.exit(0) return self.opt \ No newline at end of file diff --git a/deepmosaic.py b/deepmosaic.py index cfc8717..1d5c4e6 100644 --- a/deepmosaic.py +++ b/deepmosaic.py @@ -68,15 +68,18 @@ def main(): print('This type of file is not supported') util.clean_tempfiles(opt, tmp_init = False) - + if __name__ == '__main__': + if opt.debug: + main() + sys.exit(0) try: main() print('Finished!') except Exception as ex: print('--------------------ERROR--------------------') print('--------------Environment--------------') - print('DeepMosaics: 0.4.0') + print('DeepMosaics: 0.5.0') print('Python:',sys.version) import torch print('Pytorch:',torch.__version__) diff --git a/docs/exe_help.md b/docs/exe_help.md index 98ee6a2..d5b96aa 100644 --- a/docs/exe_help.md +++ b/docs/exe_help.md @@ -1,7 +1,9 @@ ## DeepMosaics.exe Instructions -[[中文版]](./exe_help_CN.md) +**[[中文版]](./exe_help_CN.md)** This is a GUI version compiled in Windows.
Download this version and pre-trained model via [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs) [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ)
+Video tutorial => [[youtube]](https://www.youtube.com/watch?v=1kEmYawJ_vk) [[bilibili]](https://www.bilibili.com/video/BV1QK4y1a7Av)
+ Attentions:
- Require Windows_x86_64, Windows10 is better.
@@ -9,11 +11,29 @@ Attentions:
- Run time depends on computer performance.
- If output video cannot be played, you can try with [potplayer](https://daumpotplayer.com/download/).
- GUI version update slower than source.
+ +### How to install +#### CPU version +* 1.Download and install Microsoft Visual C++ + https://aka.ms/vs/16/release/vc_redist.x64.exe +#### GPU version +Only suppport NVidia GPU above gtx1060(Driver:above 460 & CUDA:11.0) +* 1.Download and install Microsoft Visual C++ + https://aka.ms/vs/16/release/vc_redist.x64.exe +* 2.Update your gpu drive to 460(or above) + https://www.nvidia.com/en-us/geforce/drivers/ +* 3.Download and install CUDA 11.0: + https://developer.nvidia.com/cuda-toolkit-archive + +You can also download them on BaiduNetdisk +https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ +Password: 1x0a + ### How to use * step 1: Choose image or video. * step 2: Choose model(Different pre-trained models are suitable for different effects) -* step3: Run program and wait. -* step4: Cheek reult in './result'. +* step 3: Run program and wait. +* step 4: Cheek reult in './result'. ### Introduction to pre-trained models * Mosaic @@ -22,10 +42,10 @@ Attentions:
| :------------------------------: | :---------------------------------------------------------: | | add_face.pth | Add mosaic to all faces in images/videos. | | clean_face_HD.pth | Clean mosaic to all faces in images/video.
(RAM > 8GB). | -| add_youknow.pth | Add mosaic to all (FBI Warning) in images/videos. | -| clean_youknow_resnet_9blocks.pth | Clean mosaic to all (FBI Warning) in images/videos. | -| clean_youknow_video.pth | Clean mosaic to all (FBI Warning) in videos. | -| clean_youknow_video_HD.pth | Clean mosaic to all (FBI Warning) in videos.
(RAM > 8GB) | +| add_youknow.pth | Add mosaic to ... in images/videos. | +| clean_youknow_resnet_9blocks.pth | Clean mosaic to ... in images/videos. | +| clean_youknow_video.pth | Clean mosaic to ... in videos. It is better for processing video mosaics | + * Style Transfer @@ -50,8 +70,8 @@ Attentions:
* 7. More options can be input. * 8. Run program. * 9. Open help file. -* 10. Sponsor our project. -* 11. Version information. +* 10. Sponsor our project. +* 11. Version information. * 12. Open the URL on github. ### Introduction to options @@ -60,7 +80,7 @@ If you need more effects, use '--option your-parameters' to enter what you need | Option | Description | Default | | :----------: | :----------------------------------------: | :-------------------------------------: | -| --use_gpu | if -1, do not use gpu | 0 | +| --gpu_id | if -1, do not use gpu | 0 | | --media_path | your videos or images path | ./imgs/ruoruo.jpg | | --mode | program running mode(auto/clean/add/style) | 'auto' | | --model_path | pretrained model path | ./pretrained_models/mosaic/add_face.pth | diff --git a/docs/exe_help_CN.md b/docs/exe_help_CN.md index a38b6e5..8ec4b06 100644 --- a/docs/exe_help_CN.md +++ b/docs/exe_help_CN.md @@ -1,18 +1,39 @@ ## DeepMosaics.exe 使用说明 下载程序以及预训练模型 [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs) [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ)
+[视频教程](https://www.bilibili.com/video/BV1QK4y1a7Av)
+ 注意事项:
- - 程序的运行要求在64位Windows操作系统,我仅在Windows10运行过,其他版本暂未经过测试
+ + - 程序的运行要求在64位Windows操作系统,我们仅在Windows10运行过,其他版本暂未经过测试
- 请根据需求选择合适的预训练模型进行测试
- - 运行时间取决于电脑性能,对于视频文件,我们建议使用源码以及GPU运行
+ - 运行时间取决于电脑性能,对于视频文件,我们建议使用GPU运行
- 如果输出的视频无法播放,这边建议您尝试[potplayer](https://daumpotplayer.com/download/).
- 相比于源码,该版本的更新将会延后. +### 如何安装 +#### CPU version +* 1.下载安装 Microsoft Visual C++ + https://aka.ms/vs/16/release/vc_redist.x64.exe +#### GPU version +仅支持gtx1060及以上的NVidia显卡(要求460版本以上的驱动以及11.0版本的CUDA, 注意只能是11.0) +* 1.Download and install Microsoft Visual C++ + https://aka.ms/vs/16/release/vc_redist.x64.exe +* 2.Update your gpu drive to 460(or above) + https://www.nvidia.com/en-us/geforce/drivers/ +* 3.Download and install CUDA 11.0: + https://developer.nvidia.com/cuda-toolkit-archive + +当然这些也能在百度云上下载 +https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ +提取码: 1x0a + ### 如何使用 + * step 1: 选择需要处理的图片或视频 * step 2: 选择预训练模型(不同的预训练模型有不同的效果) -* step3: 运行程序并等待 -* step4: 查看结果(储存在result文件夹下) +* step 3: 运行程序并等待 +* step 4: 查看结果(储存在result文件夹下) ## 预训练模型说明 当前的预训练模型分为两类——添加/移除马赛克以及风格转换. @@ -23,10 +44,10 @@ | :------------------------------: | :-------------------------------------------: | | add_face.pth | 对图片或视频中的脸部打码 | | clean_face_HD.pth | 对图片或视频中的脸部去码
(要求内存 > 8GB). | -| add_youknow.pth | 对图片或视频中的十八禁内容打码 | -| clean_youknow_resnet_9blocks.pth | 对图片或视频中的十八禁内容去码 | -| clean_youknow_video.pth | 对视频中的十八禁内容去码 | -| clean_youknow_video_HD.pth | 对视频中的十八禁内容去码
(要求内存 > 8GB) | +| add_youknow.pth | 对图片或视频中的...内容打码 | +| clean_youknow_resnet_9blocks.pth | 对图片或视频中的...内容去码 | +| clean_youknow_video.pth | 对视频中的...内容去码,推荐使用带有'video'的模型去除视频中的马赛克 | + * 风格转换 @@ -52,8 +73,8 @@ * 7. 自行输入更多参数，详见下文 * 8. 运行 * 9. 打开帮助文件 -* 10. 支持我们 -* 11. 版本信息 +* 10. 支持我们 +* 11. 版本信息 * 12. 打开项目的github页面 ### 参数说明 @@ -62,7 +83,7 @@ | 选项 | 描述 | 默认 | | :----------: | :------------------------: | :-------------------------------------: | -| --use_gpu | if -1, do not use gpu | 0 | +| --gpu_id | if -1, do not use gpu | 0 | | --media_path | 需要处理的视频或者照片的路径 | ./imgs/ruoruo.jpg | | --mode | 运行模式(auto/clean/add/style) | 'auto' | | --model_path | 预训练模型的路径 | ./pretrained_models/mosaic/add_face.pth | @@ -75,7 +96,7 @@ | --mosaic_mod | 马赛克类型 -> squa_avg/ squa_random/ squa_avg_circle_edge/ rect_avg/random | squa_avg | | --mosaic_size | 马赛克大小，０则为自动 | 0 | | --mask_extend | 拓展马赛克区域 | 10 | -| --mask_threshold | 马赛克区域识别阈值 0~255 | 64 | +| --mask_threshold | 马赛克区域识别阈值 0~255,越小越容易被判断为马赛克区域 | 64 | * 去除马赛克 diff --git a/docs/options_introduction.md b/docs/options_introduction.md index 1947c97..3888eed 100644 --- a/docs/options_introduction.md +++ b/docs/options_introduction.md @@ -5,7 +5,7 @@ If you need more effects, use '--option your-parameters' to enter what you need | Option | Description | Default | | :----------: | :------------------------: | :-------------------------------------: | -| --use_gpu | if -1, do not use gpu | 0 | +| --gpu_id | if -1, do not use gpu | 0 | | --media_path | your videos or images path | ./imgs/ruoruo.jpg | | --start_time | start position of video, default is the beginning of video | '00:00:00' | | --last_time | limit the duration of the video, default is the entire video | '00:00:00' | diff --git a/docs/options_introduction_CN.md b/docs/options_introduction_CN.md index ef775a1..8557d04 100644 --- a/docs/options_introduction_CN.md +++ b/docs/options_introduction_CN.md @@ -5,7 +5,7 @@ | 选项 | 描述 | 默认 | | :----------: | :------------------------: | :-------------------------------------: | -| --use_gpu | if -1, do not use gpu | 0 | +| --gpu_id | if -1, do not use gpu | 0 | | --media_path | 需要处理的视频或者照片的路径 | ./imgs/ruoruo.jpg | | --start_time | 视频开始处理的位置，默认从头开始 | '00:00:00' | | --last_time | 处理的视频时长，默认是整个视频 | '00:00:00' | diff --git a/docs/pre-trained_models_introduction.md b/docs/pre-trained_models_introduction.md index 2c4a3f3..cc74728 100644 --- a/docs/pre-trained_models_introduction.md +++ b/docs/pre-trained_models_introduction.md @@ -10,8 +10,8 @@ Download pre-trained model via [[Google Drive]](https://drive.google.com/open?i | clean_face_HD.pth | Clean mosaic to faces in images/video.
(RAM > 8GB). | | add_youknow.pth | Add mosaic to ... in images/videos. | | clean_youknow_resnet_9blocks.pth | Clean mosaic to ... in images/videos. | -| clean_youknow_video.pth | Clean mosaic to ... in videos. | -| clean_youknow_video_HD.pth | Clean mosaic to ... in videos.
(RAM > 8GB) | +| clean_youknow_video.pth | Clean mosaic to ... in videos. It is better for processing video mosaics | + ### Style Transfer diff --git a/docs/pre-trained_models_introduction_CN.md b/docs/pre-trained_models_introduction_CN.md index 9156391..9b82e2a 100644 --- a/docs/pre-trained_models_introduction_CN.md +++ b/docs/pre-trained_models_introduction_CN.md @@ -10,8 +10,8 @@ | clean_face_HD.pth | 对图片或视频中的脸部去码
(要求内存 > 8GB). | | add_youknow.pth | 对图片或视频中的...内容打码 | | clean_youknow_resnet_9blocks.pth | 对图片或视频中的...内容去码 | -| clean_youknow_video.pth | 对视频中的...内容去码 | -| clean_youknow_video_HD.pth | 对视频中的...内容去码
(要求内存 > 8GB) | +| clean_youknow_video.pth | 对视频中的...内容去码,推荐使用带有'video'的模型去除视频中的马赛克 | + ### 风格转换 diff --git a/docs/training_with_your_own_dataset.md b/docs/training_with_your_own_dataset.md index fa36938..bc2a30c 100644 --- a/docs/training_with_your_own_dataset.md +++ b/docs/training_with_your_own_dataset.md @@ -10,7 +10,11 @@ We will make "face" as an example. If you don't have any picture, you can downlo - [Pytorch 1.0+](https://pytorch.org/) - NVIDIA GPU(with more than 6G memory) + CUDA CuDNN
#### Dependencies -This code depends on opencv-python, torchvision, matplotlib available via pip install. +This code depends on opencv-python, torchvision, matplotlib, tensorboardX, scikit-image available via conda install. +```bash +# or +pip install -r requirements.txt +``` #### Clone this repo ```bash git clone https://github.com/HypoX64/DeepMosaics @@ -32,31 +36,31 @@ python draw_mask.py --datadir 'dir for your pictures' --savedir ../datasets/draw python get_image_from_video.py --datadir 'dir for your videos' --savedir ../datasets/video2image --fps 1 ``` ### Clean mosaic dataset -We provide several methods for generating clean mosaic datasets. However, for better effect, we recommend train a addmosaic model in a small data first and use it to automatically generate datasets in a big data.(recommend: Method 2(for image) & Method 4(for video)) -* Method 1: Use drawn mask to make pix2pix(HD) datasets(Require``` origin_image``` and ```mask```) +We provide several methods for generating clean mosaic datasets. However, for better effect, we recommend train a addmosaic model in a small data first and use it to automatically generate datasets in a big data. (recommend: Method 2(for image) & Method 4(for video)) +* Method 1: Use drawn mask to make pix2pix(HD) datasets (Require``` origin_image``` and ```mask```) ```bash python make_pix2pix_dataset.py --datadir ../datasets/draw/face --hd --outsize 512 --fold 1 --name face --savedir ../datasets/pix2pix/face --mod drawn --minsize 128 --square ``` -* Method 2: Use addmosaic model to make pix2pix(HD) datasets(Require addmosaic pre-trained model) +* Method 2: Use addmosaic model to make pix2pix(HD) datasets (Require addmosaic pre-trained model) ```bash python make_pix2pix_dataset.py --datadir 'dir for your pictures' --hd --outsize 512 --fold 1 --name face --savedir ../datasets/pix2pix/face --mod network --model_path ../pretrained_models/mosaic/add_face.pth --minsize 128 --square --mask_threshold 128 ``` -* Method 3: Use Irregular Masks to make pix2pix(HD) datasets(Require [Irregular Masks](https://nv-adlr.github.io/publication/partialconv-inpainting)) +* Method 3: Use Irregular Masks to make pix2pix(HD) datasets (Require [Irregular Masks](https://nv-adlr.github.io/publication/partialconv-inpainting)) ```bash python make_pix2pix_dataset.py --datadir 'dir for your pictures' --hd --outsize 512 --fold 1 --name face --savedir ../datasets/pix2pix/face --mod irregular --irrholedir ../datasets/Irregular_Holes_mask --square ``` -* Method 4: Use addmosaic model to make video datasets(Require addmosaic pre-trained model. This is better for processing video mosaics) +* Method 4: Use addmosaic model to make video datasets (Require addmosaic pre-trained model. This is better for processing video mosaics) ```bash -python make_video_dataset.py --datadir 'dir for your videos' --model_path ../pretrained_models/mosaic/add_face.pth --mask_threshold 96 --savedir ../datasets/video/face +python make_video_dataset.py --model_path ../pretrained_models/mosaic/add_face.pth --gpu_id 0 --datadir 'dir for your videos' --savedir ../datasets/video/face ``` ## Training ### Add ```bash cd train/add -python train.py --use_gpu 0 --dataset ../../datasets/draw/face --savename face --loadsize 512 --finesize 360 --batchsize 16 +python train.py --gpu_id 0 --dataset ../../datasets/draw/face --savename face --loadsize 512 --finesize 360 --batchsize 16 ``` ### Clean -* For image datasets(generated by ```make_pix2pix_dataset.py```) +* For image datasets (generated by ```make_pix2pix_dataset.py```) We use [pix2pix](https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix) or [pix2pixHD](https://github.com/NVIDIA/pix2pixHD) to train model. We just take pix2pixHD as an example. ```bash git clone https://github.com/NVIDIA/pix2pixHD @@ -64,10 +68,10 @@ cd pix2pixHD pip install dominate python train.py --name face --resize_or_crop resize_and_crop --loadSize 563 --fineSize 512 --label_nc 0 --no_instance --dataroot ../datasets/pix2pix/face ``` -* For video datasets(generated by ```make_video_dataset.py```) +* For video datasets (generated by ```make_video_dataset.py```) ```bash cd train/clean -python train.py --dataset ../../datasets/video/face --savename face --savefreq 100000 --gan --hd --lr 0.0002 --lambda_gan 1 --use_gpu 0 +python train.py --dataset ../../datasets/video/face --savename face --n_blocks 4 --lambda_GAN 0.01 --loadsize 286 --finesize 256 --batchsize 16 --n_layers_D 2 --num_D 3 --n_epoch 200 --gpu_id 4,5,6,7 --load_thread 16 ``` ## Testing -Put saved network to ```./pretrained_models/mosaic/``` and rename it as ```add_face.pth``` or ```clean_face_HD.pth``` or ```clean_face_video_HD.pth``` +Put saved network to ```./pretrained_models/mosaic/``` and rename it as ```add_face.pth``` or ```clean_face_HD.pth``` or ```clean_face_video_HD.pth```and then run ```deepmosaic.py --model_path ./pretrained_models/mosaic/your_model_name``` diff --git a/imgs/icon.jpg b/imgs/icon.jpg deleted file mode 100644 index 672f6ed..0000000 Binary files a/imgs/icon.jpg and /dev/null differ diff --git a/imgs/logo.ico b/imgs/logo.ico new file mode 100644 index 0000000..75edca6 Binary files /dev/null and b/imgs/logo.ico differ diff --git a/imgs/logo.png b/imgs/logo.png new file mode 100644 index 0000000..41b8ee3 Binary files /dev/null and b/imgs/logo.png differ diff --git a/imgs/logo_withwords.png b/imgs/logo_withwords.png new file mode 100644 index 0000000..cc3290d Binary files /dev/null and b/imgs/logo_withwords.png differ diff --git a/make_datasets/csv/stars_name.csv b/make_datasets/csv/stars_name.csv deleted file mode 100644 index 4c0015b..0000000 --- a/make_datasets/csv/stars_name.csv +++ /dev/null @@ -1 +0,0 @@ -邓紫棋,范玮琪,赫子铭,邵美琪,李易峰,尤浩然,张丰毅,李宗翰,范冰冰,朱亚文,彭丹,蔡琴,杨幂,刘嘉玲,童蕾,张丹峰,Angelababy,闫妮,热依扎,柯受良,唐嫣,张静初,林忆莲,濮存昕,刘诗诗,莫小棋,潘粤明,贾晓晨,周星驰,毛俊杰,万茜,隋俊波,李小璐,赵文卓,潘长江,毛孩,柳岩,应采儿,郑秀文,吕丽萍,周杰伦,聂远,萧淑慎,安雅萍,赵丽颖,毛晓彤,宣萱,黄小柔,张柏芝,张智尧,何美钿,涂黎曼,赵本山,黄秋生,张铁林,印小天,刘德华,唐一菲,于明加,张可颐,姚笛,干露露,李光洁,任容萱,高圆圆,杨丞琳,陈慧琳,余少群,谢霆锋,利智,左小青,徐怀钰,林志颖,六小龄童,郑元畅,朱媛媛,李菲儿,梁朝伟,苗侨伟,赵宝刚,吴镇宇,贾青,吴亚馨,楼南光,林志玲,郑伊健,曾志伟,吴京安,钟汉良,金城武,侯梦莎,王传一,孙俪,黎姿,吴君如,刘洲成,戚薇,张惠妹,周韵,丁小芹,林心如,言承旭,植敬雯,许冠文,房祖名,邱泽,叶玉卿,莫少聪,佟丽娅,张曼玉,辰亦儒,李李仁,吴奇隆,蓝洁瑛,陈百强,田海蓉,郭碧婷,邹兆龙,薛佳凝,戚美珍,李连杰,蒋勤勤,谷智鑫,吴廷烨,汤唯,吴绮莉,朱梓骁,丁春诚,马伊琍,钟丽缇,马景涛,张博宇,陆毅,张嘉倪,狄龙,马浚伟,周润发,金世佳,何超仪,冯宝宝,陈乔恩,杨恭如,周渝民,王思平,吴尊,卓依婷,朱一龙,龙飘飘,陈妍希,欧弟,张雨生,张雅蓓,陈冠希,陈建州,陶慧敏,官晶华,杜汶泽,梁家辉,吕颂贤,衣珊,王祖贤,袁咏仪,张译,鲍国安,伊能静,钟嘉欣,袁洁莹,罗乐林,张学友,李依晓,杨童舒,曹炳琨,罗志祥,杨乐乐,李心洁,姜昕言,赵又廷,刘若英,秦沛,朱铁,王祖蓝,李嘉欣,隋棠,丁嘉丽,吴秀波,颜丹晨,李诗韵,谭耀文,马天宇,莫文蔚,郑则仕,金玉婷,张耀扬,元彪,贺军翔,涂松岩,娄艺潇,徐熙娣,斯琴高丽,萧正楠,蔡依林,杨玏,王君平,王一楠,林青霞,张含韵,程愫,唐笑,彭于晏,蒋劲夫,王茜华,程琤,王丽坤,王若涵,王小利,狄莺,孙红雷,袁弘,卓文萱,郑俊弘,甄子丹,安志杰,冯淬帆,刘畊宏,李小冉,袁立,汤镇业,傅明宪,徐熙媛,余男,闫学晶,吴建飞,马苏,薛凯琪,于小伟,周秀娜,张俪,梁小龙,林子祥,沈卓盈,林依晨,庾澄庆,郭晋安,赵文瑄,巨兴茂,刘晓洁,徐帆,黄子扬,徐若瑄,曾沛慈,董骠,夏一瑶,叶一茜,沈佳妮,陈展鹏,刘思言,张歆艺,宋丹丹,蔡康永,赖琳恩,释小龙,潘玮柏,蒲巴甲,杨钫涵,汪东城,斓曦,杨佑宁,辛柏青,吴映洁,焦恩俊,陈法蓉,颜仟汶,王珞丹,马雅舒,王力可,陆树铭,张智霖,吕一,天心,崔漫莉,关之琳,阮经天,迟重瑞,邬倩倩,江疏影,翁美玲,曾恺玹,寺唯宏正,舒淇,王学兵,刘蓓,丁子峻,张嘉译,钱小豪,王雅捷,邵兵,颖儿,黄磊,郑浩南,王若心,陈意涵,林熙蕾,于荣光,钮承泽,安以轩,林雅诗,小小彬,刘劲,李佳航,蔡少芬,刘品言,王一淼,霍思燕,何润东,张震岳,邢佳栋,徐静蕾,张凯丽,温兆伦,利昂霖,吴京,童瑶,林依轮,陈彦妃,蔡卓妍,徐正曦,何家劲,黎耀祥,吴彦祖,姚星彤,江祖平,谢金燕,赵雅芝,姚芊羽,李保田,叶蕴仪,田朴珺,欧阳震华,马少骅,张艾嘉,尹天照,潘霜霜,韩雨芹,吴家丽,张韶涵,唐禹哲,戚迹,毛舜筠,梅艳芳,侯佩岑,陶晶莹,班嘉佳,贾静雯,钟楚红,邬君梅,谢君豪,俞飞鸿,余文乐,周晓涵,谭松韵,张卫健,关悦,蓝盈莹,曾宝仪,炎亚纶,秦海璐,唐诗咏,方中信,李亚鹏,许冠杰,张文慈,马蹄露,谭咏麟,高露,杨一展,苑琼丹,苏有朋,钟镇涛,徐洪浩,王子子,周海媚,万梓良,吕良伟,杨泽霖,陈若仪,汤加丽,杨明娜,张晨光,陈数,陈志朋,张涵予,赵子惠,郭富城,茹萍,徐熙颜,李幼斌,陈怡蓉,郭珍霓,曹达华,田蕊妮,寇振海,姜大卫,赵子琪,赵传,曾之乔,薛芷伦,寇乃馨,李宇春,张靓颖,周笔畅,何洁,刘亦菲,陈好,尚雯婕,张筱雨,韩雪,孙菲菲,陈紫函,朱雅琼,江一燕,厉娜,许飞,胡灵,郝菲尔,刘力扬,reborn,章子怡,谭维维,魏佳庆,张亚飞,李旭丹,孙艺心,巩贺,艾梦萌,闰妮,王蓉,汤芳,牛萌萌,赵薇,周迅,金莎,纪敏佳,黄雅莉,阿桑,董卿,金铭,徐行,朱妍,夏颖,陈西贝,冯家妹,高娅媛,林爽,郑靖文,陶虹,黄奕,董洁,巩俐,于娜,孟广美,Gameapple,美女奉奉,小龙女彤彤,张子萱,果子,丁贝莉,吸血芭比,公交MM,香香,段思思,二月丫头,刘羽琦,dodolook,拉拉公主,沈丽君,周璟馨,丁叮,谢雅雯,陈嘉琪,宋琳,郭慧敏,卢洁云,佘曼妮,黄景,马艳丽,蒋雯丽,宁静,许晴,瞿颖,张延,闵春晓,蔡飞雨,邓莎,白冰,程媛媛,吴婷,殷叶子,朱伟珊,孙菂,赵梦恬,龚洁,许晚秋,杨舒婷,乔维怡,王海珍,易慧,谢雨欣,陈娟红,舒畅,曹颖,李冰冰,王艳,沈星,阿朵,周洁,杨林,李霞,陈自瑶,李湘,金巧巧,梅婷,刘涛,安又琪,杨钰莹,马伊俐,陈红,鲍蕾,牛莉,胡可,龚蓓苾,田震,吕燕,王姬,苗圃,李欣汝,王小丫,秦岚,彭心怡,邓婕,眉佳,李媛媛,刘晓庆,杨若兮,黄圣依,林熙,斯琴格日乐,宋祖英,郝蕾,乐珈彤,冯婧,盖丽丽,杨澜,沈冰,王宇婕,王希维,姜培琳,何晴,焦媛,白灵,胡静,陈冲,刘怡君,韦唯,龚雪,周彦宏,刘丹,傅艺伟,谢东娜,黑鸭子,周璇,杨欣,陈小艺,伍宇娟,苏瑾,李玲玉,潘虹,沈丹萍,岳红,赵静怡,宋晓英,梁咏琪,李纹,林嘉欣,周丽淇,潘伟柏,梁静茹,周慧敏,杨千桦,林俊杰,孙燕姿,杜雯惠,郑少秋,柯有伦,麦兆辉,林咏伦,苏友朋,李志刚,杜德伟,方力申,刘日曦,刘雅丽,陈炜,孙佳君,崔健邦,陈秀雯,郭霭明,樊亦敏,黄倩,张小娴,袁文杰,连凯,刘伟强,许志安,彭嘉丽,区文诗,汤盈盈,侯湘婷,张慧仪,梁韵蕊,陈颖研,李婉华,许鞍华,向海岚,丘凯敏,刘晓彤,文颂娴,刘心悠,廖安丽,胡彦斌,黄耀明,伦永亮,夏萍,陈芷菁,朱茵,关德辉,韩君婷,张曼伶,林国斌,黄智贤,彭健新,陈浩民,廖碧儿,胡林,梁绰妍,邓蔼霖,八两金,陈文媛,赵学而,林保怡,黄宗泽,黄泽锋,周国丰,李国豪,李小龙,梁家仁,邝美云,蔡一智,卢巧音,车淑梅,梁芷珊,刘钖明,李若彤,郑裕玲,林莉,彭羚,洪欣,徐子淇,叶童,施念慈,嘉碧仪,李心媚,陈加玲,蔡卓研,司徒瑞祈,应昌佑,杨采妮,畲诗曼,李思欣,陈绮贞,杨雪仪,江芷妮,陈采岚,林一峰,潘芝莉,欧阳妙芝,黄凯芹,锺丽缇,活己岚,欧倩怡,姚嘉妮,李玟,蒋怡,马小灵,陈宝珠,缪骞人,吴家乐,罗启新,陈敏婷,王心凌,胡凯欣,刘松仁,陈晓东,李修贤,夏韶声,王杰,胡大为,吴大维,陶吉吉,卢戴维,曾建明,黄子华,吴浩康,何韵诗,叶德娴,麦子豪,张国洪,李浩林,郭政鸿,李迪文,夏健龙,关浩扬,陈海恒,李逸朗,张伟文,唐剑康,朱洁仪,陈嘉熙,陈奕迅,沈傲君,张兆辉,吴国敬,梁汉文,苏志威,姚莹莹,何绮玲,姚子羚,孙泳恩,江美仪,陈洁灵,李美凤,江希文,廖隽嘉,陈奕斯,官恩娜,陈嘉桦,蔡雪敏,陈雅伦,卢凯彤,蔡子健,邓肇欣,萧亚轩,倪晨曦,林二汶,周华健,卢淑仪,关宝慧,黄伊汶,张锦程,周国贤,葛民辉,巫启贤,孙耀威,伍家廉,郭伟亮,李天翔,李敏,袁弥明,关咏荷,陈嘉容,麦包,许慧欣,陈法拉,王菲,黄日华,活希儿,袁彩云,陈慧珊,张天爱,郭少芸,叶丽仪,陈勋奇,李进,海俊杰,罗嘉良,唐奕聪,林苑,锺沛枝,黄淑仪,杨其龙,锺欣桐,陈素莹,利嘉儿,陈嘉上,叶佩雯,李克勤,谭小环,徐濠萦,刘恺威,田馥甄,朱凯婷,欧海伦,马海伦,谭玉瑛,陈玉莲,吕怡慧,温碧霞,黄泆潼,梁佩瑚,陈逸宁,梁洛施,寇鸿萍,王菀之,成龙,陈晓琪,王树熹,吴宗宪,张洁莲,吴倩莲,梁咏琳,关心妍,黄沾,林海峰,李嘉文,卫兰,马诗慧,马阅,张咏恩,曾仕贤,朱永棠,梁琤,房祖明,张信哲,刘镇伟,陈汉诗,甄妮,张洪量,林晓峰,李汉文,卫诗,詹瑞文,少爷占,马楚成,谷祖琳,岑宝儿,唐丽球,陈佩珊,马伟豪,倪震,马德钟,郑敬基,陈任,容祖儿,邓建明,梁荣忠,万绮雯,蔡慧敏,吴宇森,姜戴维,杜琪峰,邓兆尊,颜国梁,阮德锵,李思捷,邓一君,陈苑琪,蔡依琳,李中希,陈小春,萧芳芳,陈彦行,李乐诗,郑欣宜,盖鸣辉,滕丽名,高丽虹,夏利奥,陈琪,麦浚龙,侧田,汤宝如,吴雨霏,郁礼贤,徐子珊,周汶锜,梁佩诗,谢安琪,蔡枫华,朱孝天,吴克群,王合喜,李静怡,郭可盈,马国明,陈启泰,卢惠光,锺镇涛,关智斌,郑嘉颖,林子瑄,商天娥,陈洁仪,麦洁文,阮小仪,王家敏,梁雪湄,周俊伟,吴启华,郑丹瑞,李亚男,唐宁,古巨基,詹志文,黎明,张国荣,陈慧明,何俐恩,李莉莉,康子妮,王馨平,锺嘉欣,汪明荃,郭静,李丽珍,朱玲玲,罗莉萨,古天乐,阮兆祥,樊少皇,张继聪,苏玉华,卢冠廷,盖世宝,沈殿霞,陈美诗,刘思惠,傅佩嘉,蒋雅文,曹敏莉,王秀琳,霍纹希,陈曼娜,许秋怡,魏骏杰,曾华倩,陈妙瑛,锺丽淇,原子鏸,雷颂德,郭耀明,陈霁平,肥妈,刘家聪,张玛莉,韩马利,邹凯光,高钧贤,范晓萱,许美静,罗美薇,吴婉芳,邝文珣,赖雅妍,吴美珩,谢天华,王敏德,周文健,梁继璋,唐文龙,陶大宇,王光良,杨紫琼,叶璇,郑雪儿,米雪,伏明霞,张佳佳,朱健钧,杨爱谨,罗敏庄,朱咪咪,杨千嬅,卢敏仪,董敏莉,陈法容,陈豪,胡杏儿,陈松伶,兰茜,薛家燕,胡定欣,陈百祥,黄纪莹,吴文忻,沈玉薇,谭凯欣,张燊悦,张家辉,林中定,周丽琪,郑文雅,吴佩慈,胡蓓蔚,黄柏高,刘彩玉,刘绰琪,邓建泓,雷有晖,黄贯中,锺保罗,方保罗,谢伟俊,徐小凤,陈宝莲,杨宝玲,任葆琳,陈可辛,黎彼得,文彼得,蓝奕邦,陈国邦,刘美君,朱慧敏,陈莉敏,黄婉伶,傅天颖,郭芯其,陈建颖,林峰,黄百鸣,蔡济文,曹永廉,许同恩,杨洛婷,蔡一杰,戴梦梦,范振锋,许冠英,林振强,任贤齐,韦绮珊,郑健乐,梁奕伦,罗文,吴卓羲,郑中基,于仁泰,黄婉佩,周永恒,姚乐碧,黄卓玲,蔡洁雯,叶蒨文,李璨森,陈宇琛,洪金宝,陈键锋,森美,林青峰,杨英伟,邰正宵,刘永健,林姗姗,任家萱,李丽蕊,王卉,刘青云,周影,谢宁,刘婉君,张敏,陈敏芝,郭秀云,陈淑兰,邓萃雯,邓上文,姚乐怡,张玉珊,关淑怡,杨思琦,张晓,关秀媚,任达华,雷宇扬,伍思凯,梁思浩,李嘉慧,郭羡妮,唐季礼,黄湘怡,冯德伦,颜福伟,陈志云,车沅沅,黎明诗,郑融,王从希,邓丽欣,黄家强,张致恒,黎瑞莲,陈秀茹,陈锦鸿,杨怡,傅颖,李绮红,李茏怡,林敏俐,谢宛婷,洪天明,何宝生,陈德彰,林祖辉,梁靖琪,阮民安,叶翠翠,黄宇诗,周嘉玲,范逸臣,邓颖芝,吴建豪,梁慧嘉,邹琛玮,陈司翰,唐韦琪,蔡淇俊,谷德昭,王浩信,王贤志,陈德容,李明慧,黎瑞恩,骆乐,梁小冰,杨秀惠,雷凯欣,锺汉良,翁慧德,莫华伦,韩毓霞,刘浩龙,苏永康,孔庆翔,陈自强,林颖娴,张咏妍,刘小慧,俞琤,杨婉仪,梁敏仪,黄伟文,唐尧麟,张茵,甄楚倩,蒙嘉慧,郑希怡,刘文娟,谭凯琪,苍井空,川滨奈美,堤莎也加,町田梨乃,二阶堂仁美,饭岛爱,饭田夏帆,饭冢友子,芳本叶月,冈崎结由,冈田丽奈,高木萌美,高田礼子,高原流美,宫本真美,宫岛司,古都光,光月夜也,河村亚季子,河井梨绪,黑崎扇菜,红月流奈,华歌恋,吉川萌,及川奈央,吉川真奈美,吉崎纱南,吉野莎莉,今井明日香,今木翔子,金泽蓝子,进藤玲菜,井上可奈,久保美希,酒井未希,臼井利奈,菊池丽香,菊池英里,菊池智子,橘真央,具志坚阳子,可爱亚织沙,葵小夏,蓝山南,兰望美,里见奈奈子,里美奈奈子,里美由梨香,立花丽华,立木爱,凉白舞,铃川玲理,铃江纹奈,铃木麻奈美,芦屋瞳,麻川美绪,麻生叶子,美里霞,美崎凉香,美雪沙织,美月莲,明日香,木谷麻耶,奈奈见沙织,内藤花苗,内田理沙,鲇川亚美,片濑亚纪,平山朝香,前原优树,前原佑子,浅见伽椰,浅井理,青木琳,青木玲,青野诗织,青羽未来,青沼知朝,秋本玲子,秋菜里子,秋元优奈,如月可怜,若林树里,若月树里,森下理音,纱月结花,杉浦清香,杉山亚来,山下由美子,杉原凉子,上原留华,神城千佳,神崎麻衣,神崎麻子,矢吹丽,手束桃,树本凉子,水城凛,水野朋美,水野茜,水越丽子,四季彩香寺田弥生,松浦梦,松浦唯,松田千夏,松下爱来,松下可怜,松元静香,速水真保,藤彩香,藤代流花,藤崎秋,藤森智子,天衣美津,田村麻衣,望月瞳,舞岛美织,午后野弥生,西泽友里,夏美舞,相川未希,相户爱,相田由美,小仓杏,小川流果,筱宫知世,小栗杏菜,小森美王,小室优奈,小野由佳,筱原凉,小泽菜穗,小泽玛丽亚,筱冢真树子,星爱丽斯,星崎瞳,星野绫香,星野洋子,星野真弥,徐若樱,雪乃小春,岩下美季,遥优衣,野宫美忧,野原奈津美,叶月千穗,伊东美华,一色丽矢,一色鲇美,一条沙希,乙伊沙也加,樱井沙也加,由树莉莉,有川真生,有吉奈生子,有森玲香,雨宫优衣,原千寻,原史奈,原田春奈,远野麻耶,月野静玖,早纪麻未,早乙女舞,泽舞音,长濑爱,长月亚美,真木亚里沙,真山润,中川珠代,中村理央,中根由真,中山美玖,中原绫,仲井美帆,竹田树理,佐伯美奈,佐佐木,幸田梨纱,北原爱子,成膳任,戴文青木,德永千奈美,笛木优子,福原爱,高见美香,高树玛丽亚,宫崎葵,观月雏乃,海江田纯子,后藤理莎,后藤香南子,矶山沙耶香,矶山沙也加,吉冈美穗,吉泽瞳,加纳则子,加藤小雪,菅谷梨沙子,结城翼,井上和香,井上熏,酒井瑛里,久纱野水萌,铃木爱理,玲木美生,泷泽乃南,美依旗由美,木下亚由美,前田知惠,前原爱,浅田真央,清水佐纪,入江纱绫,三尺真奈美,三宅尚子,森下千里,上原绫,石村舞波,矢田亚希子,市川由衣,市井纱耶香,嗣永桃子,松岛菜菜子,松居彩,松元莉绪,樋口真未,细川直美,夏烧雅,相乐纪子,小川熏,小林惠美,小野奈美,小泽真珠,星野亚希,须藤茉麻,亚纪奈,岩田小百合,伊藤步,优香,友崎玲,中泽裕子,佐藤麻纱,安藤沙耶,奥山唯子,白崎令于,柏木奈纯,板谷佑,滨田翔子,朝雾唯,川崎爱,大和抚子,大西由梨香,岛田百合花,二宫优,绀野舞子,岗原厚子,高鸠阳子,古河由摩,谷田未央,河合绫纯美,和久井辛,和希沙,黑田美礼,横仓里奈,后藤亚维梨,户田惠梨香,吉濑美智子,加藤麻依,江纱绫,井上诗织,井上优香,井真理绘,堀井美月,莲沼民子,柳明日,落合玲奈,牧濑奈美,木下亚由,奈良沙绪理,浅丘南,秋本那夜,秋山玲子,秋庭比吕子,三尺真奈,三井保奈美,森下真理,山吹美花,山口纱弥加,杉里香,神代弓子,树梨沙,水谷利加,松鸠永里奈,松山麻美,松屿初音,塔山直美,藤香南子,天使美树,天野理惠,田崎由希,桐岛淳子,尾崎美果,西野美绪,相泽优香,小林里实,小早川洋子,叶山小姬,樱树露衣,樱田佳子,永井绘理香,远藤真纪,早川美波,早川桃香,折原琴,中鸠广香,中泽夏木,竹野内丰,佐藤江梨花,爱内萌,爱田露美,爱田毛毛,安倍夏实,安原真美,奥山香,八木泽,白川美奈美,白鸟智惠子,白亚朱里,北山静香,北原步,北原真裕,仓本安奈,仓田杏里,朝比奈真理,朝仓加穗里,朝仓玛丽亚,持田茜,冲田由加里,川奈由依,大友梨奈,岛田香奈,堤沙也加,渡边香,风间舞,风见京子,福山洋子,冈本夏生,高仓梨奈,高野瞳,宫本瞳,宫本阳子,宫地奈,宫间沙布子,工藤瞳,宫下杏菜,河田纯子,和希结衣,横山千枝,华美月,姬野莉梦,吉田友里,吉野碧,菅野美寿纪,江口美贵,结衣美沙,金城美丽,井上彩菜,井上雅,鸠村熏,酒井美幸,菊池则江,君岛美香,可爱亚芝莎,来栖敦子,蓝原夕妃,蛯原舞,立花优,立原贵美,恋野恋,铃木由美,麻生岬,麻生真美子,麻田子,茂森亚弓,美波志保,木内亚吉拉,内田京香,品田由依,平山绫,前岛美步,前田亚纪,浅见怜,浅野子,青木美里,清木裕子,青山遥,青山叶子,三濑真美子,三崎真绪,三上夕希,三尾安齐,森宏子,森山子,森野雫,山口理惠,山口美纪,山口萌,杉山圭,杉田惠美,山田誉子,杉原爱砂,上原亚也加,神崎美树,神田美沙纪,圣瑛麻,石川恩惠,石川瞳,石川优季奈,矢吹凉子,矢吹麻理奈,矢择优子,水城梓,水希遥,松岛奈奈子,松纱良,速水怜,藤井彩,藤崎弭代,田山真美子,田中梨子,田中美久,瞳理欧,望月七,望月沙香,望月英子,武田真治,夏结玲,相纪美,相乐晴子,相仁泽美,相原里奈,翔巴辉,小池亚弭,小峰由衣,筱宫庆子,小田凉子,小向杏奈,小野濑,幸田李梨,岩崎千鹤,野宫凛子,野野由利加,叶山路易,一宫理绘,伊藤美沙纪,一条小百合,樱庭凉子,永井流奈,优木里绪奈,优木美纱,羽田夕夏,原惠美子,远山雪乃,远藤梨奈,早濑理沙,早乙女香织,长谷川,真纯麻子,织原奈美,柊丽子,中条佳奈子,中野美奈,仲村桃,足立美,佐藤有纪,八木亚希子,朝比奈英里,朝仓仁美,朝仓香乃,朝仓遥,朝美光,朝美绘乃香,朝丘瞳,朝霞佳绘美,赤西凉,川野亚希子,大久保玲,饭干惠子,福光千穗,冈田安希,高以亚希子,和久井由菜,吉木纯菜,吉泽京子,井上华菜,君野梦,堀切子,楠麻耶,南使香,平丸久美子,青木友梨,仁乃庆子,三浦富美子,山本京子,榊彩弥,矢野显子,水野亚美,水泽千夏,太田优,藤森子,相崎琴音,相泽纪美,星美织,杨原京子,早川濑里奈,斋藤朝子,长泽筑实,中谷香织,中森子,中条美华,竹内爱,庄司爱,佐藤春菜,佐藤子,津野田熏,吉井玲奈,阿嘉奈津,安藤希,安田奈央,奥川希美,奥山美夏,白鸟美由纪,百合香,北村夕起,北山英里,朝仓志穗,朝丘南,朝丘纱智,朝日美穗,朝永真弥,朝长真弥,纯名梨沙,村田洋子,大林典子,二瓶有香,芳贺优里雅,妃今日子,福美香织,冈本多绪,刚野凤子,高桥由美子,桂木萌,河合梓,横须贺,吉成香,吉村优,臼井里绘,橘友贺,来栖凉子,濑户美贵子,濑雅子,林绘里,玲樱奈,美月由奈,梦野玛丽亚,秘叉华,木内美步,木尾原真弓,牧野泉,鲇川直美,清水理惠子,萩原舞,泉由香,三浦桃,松树梨,松下真依子,松元伊代,藤宫知世,田村茜,田真潮,桐岛惠理香,梶原真弓,西尾佑里,相泽朝海,相泽沙赖,小阪光,小仓艾莉丝,筱峰爱,野乃原,优里香,由津希,泽绪凛,长曾我部蓉子,真莉亚,真崎麻衣,仲根佳绘美,爱原千芙,绫波优,奥菜千春,奥菜翼,八木原麻优,白川沙也加,白石枫,白石麻梨子,宝来美雪,北原雪,宝生琉璃,草剃纯,长濑茜,赤阪梨乃,赤阪美步,大路惠美,岛津讶子,德井唯,儿岛香绪里,福泽京子,宫泽璃音,吉野美穗,橘琉璃,濑户准,濑名凉子,片濑梨音,齐藤美穗,枪田彩野,桥本美步,三笑明日香,上原绘里香,石井淳笑,松冈理穗,松井夏穗,松元亚璃沙,唐泽美树,小池绘美子,小泉琉美,小山涉,小野谷实穗,星野琉海,续丽子,岩崎美穗,泽山凉子,辰巳奈都子,热田久美,姬野香,榎本瑞穗,榎本瑞怆,榎木加奈子,星野真唯,八木泽莉央,八木泽景一,柏木瞳一郎,坂下千里子,保坂拓见,北原一咲,泷北智子,夏本加奈子,端本千奈美,爱本瑞穗,本树尤真,仓内安奈,小鹿纯子,长坂仁惠,赤坂丽,赤坂七惠,赤咲伶奈,川村美咲,春日咲衣,镰田奈津美,风见里穗,富田梨惠,高坂真由,宫咲志帆,光咲玲奈,黑坂真美,胡桃泽马里奈,吉成香子,吉田亚咲,今井优,井坂绘美,久留须由美,蓝田美丰,笠木彩花,丰岛真千子,铃木美生,吉田亚纪子,瑠川淳子,美咲沙耶,美咲亚弥,美咲亚沙,梦咲亚由,乃木真利子,乃坂绘美,佐藤和沙,金子纱香,片濑梨子,中岛千晶,浅野京子,吉泽有希子,濑间幸美,金子美铃,三田爱里,三田友穂,三咲真绪,咲小雪,沙耶香,相田纱耶香,杉田美园,侍山凉子,山崎亚美,山咲萌,山咲千里,山咲舞,山咲亚香里,山咲一步,长濑美优,长濑美华,樱井美优,坂上友香,神乐坂惠,神田朱未,神田美咲,小岛圣,泽诗奈奈子,石川施恩惠,石坂伊津佳,水咲凉子,水咲亚里美,水野理纱,松坂庆子,松坂纱良,松坂树梨,滩坂舞,藤井树,齐藤小雪,藤咲彩美,藤咲理香,白石美树,品川幸菜,吉崎渚,田代沙织,田岛美美,泽田悠里,田中玲娜,田坂菜月,田坂仁美,舞坂由衣,香坂仁见,咲田葵,薰樱子,冴岛奈绪,野坂惠美,野坂奈津美,观月亚里沙,伊吹美奈裳,音咲洵,真锅薰,冈真里子,向井真理子,松坂季实子,椎名英姬,佐仓真衣,前田亚季,坂本冬美,坂本绘,蔡妍,裴涩琪,全慧彬,宝儿,张英兰,韩彩英尹恩惠,宋慧乔,李多海,成宥利,金泰熙,金喜善,李恩珠,韩佳人,亚由美,蔡琳,bada,张力尹（chinese),李贞贤,崔智友,全智贤,张娜拉,李孝利,梁美京,文根英,林秀晶,李英爱,金静华,张瑞希,林允儿,宋允儿李秀景,郑柔美,郑多彬,简美妍,金善雅,韩智慧,韩惠珍,南相美,黄静茵,金泰妍,郑丽媛,金荷娜,刘荷娜,河利秀,孙艺珍,徐智英,何智媛,崔秀英,stephanie天舞,金贤珠,李美妍,金雅中,朴善英郑秀妍,金智秀,皇甫惠静,韩智敏,秋瓷炫,dana,朴恩惠,韩孝珠,黄美英,金正恩,申敏儿,孔孝真,金素妍,权侑莉,禹喜珍,徐珠贤,韩艺瑟,李真,高雅拉,崔明吉,李智贤,李英雅,尹海英,林智慧,李妍喜,朴喜本,甄美里,seeya,申智（高耀太）,李沇熹,金孝渊,金美淑,洪秀儿,金慧成,宋智孝,李瑶媛,朴贞雅,沈恩珍,lina上美,babyvox,崔真实,秋素英,李秀英,sunday智声,jewelry,金度完,申爱,朴信惠,金敏贞,李银珠,金南珠朴志胤,李智雅,姜恩菲,南圭丽,李青儿,高斗心,白智英,朴秀珍,朴艺珍,裴斗娜,闵先艺,赵静林,李太兰,金芝荷,李素妍,河莉秀,宣美,韩恩贞,金允慧,高恩雅,韩惠淑,沈银河,高贤贞崔茹真,李娜英,赵允熙,金莎朗,姜晶花,严正花,朴嘉熙,朴智运,闵孝琳,李宝英,玉珠铉,朴美宣,李惠淑,黄宝拉,朴幼林,吴妍秀,李姬珍,全度妍,徐智慧,李美淑,明世彬,韩艺仁,金南智,姜受延秀爱,李贤智,沈惠珍,赵贞恩,黄秀贞,钱忍和,申恩庆,洪秀贤,车贤静,张熙珍,金敏善,鲜于银淑,李荷娜,金泫雅,金孝珍,艺智苑,廉晶雅,孙泰英,上美lina,郑彩恩,贤真,金智慧,张熙真,朴素美,张真英,高小英,姜惠贞,金允珍,申爱罗,秋尚美,金喜爱,秋相美,车裕利,洪莉娜,金宝美,宋善美,李爱静,姜成妍,yuri,金惠秀,金敏喜,李清儿,郑在英,慧英,吴允儿,朴诗恩朴美善,苏幼真,李海英,崔贞润,韩银贞,金香奇,金孝真,崔志友,朴莎朗,金丽娜,孔孝珍,文素利,金圭莉,池秀媛,徐信爱,郑秀美,李敏贞,林贞恩,宋允雅,韩高恩,金贤雅,尹珍熙,崔允英,金贞兰,许伊在,韩爱莉,闵智慧,李惠英,金善儿,尹智慧,宋孝智,蔡徐坤,陈立农,范丞丞,黄明昊,林彦俊,朱正廷,王子异,王琳凯,尤长靖,毕雯珺,李希侃,黄新淳,李权哲,丁泽仁,秦奋,韩沐伯,徐鹤尼,左叶,罗正,陆定昊,董又霖,董岩磊,钱正昊,韩雍杰,木子洋,灵超,岳岳,卜凡,朱均天,朱均一,朱一文,张晨宇,应智越,万宇贤,吕晨瑜,宋微子,何东东,李长庚,张艺凡,李若天,邓烺怡,靖佩瑶,周腾阳,杨羿,张奕轩,姜京佐,许凯皓,凌崎,周彦辰,朱星杰,Rapen,Glgel,张昕,王宥辰,陈斯琪,于洁,武连杰,徐圣恩,张达源,陈名豪,王梓豪,金逸涵,甘俊,明鹏,侯浩然,余明君,姜祎,娄滋博,胡致邦,高茂桐,叶泓希,伽里,罗杰,李志杰,林浩楷,孙凡杰,于斌,何嘉庚,孙浩然,张晏恺,李俊毅,谭俊毅,黄书豪,陈义夫,闵喆祥,李让,周锐,郑瑞彬,林超泽,赵凌峰,赵俞澈,邱治谐,梁辉,杨非同,李汶翰,胡春杨,胡文煊,林陌,陈宥维,陈涛,李宗霖,嘉羿,邓泽鸣,卡斯柏,杨朝阳,邓超元,王喆,车慧轩,蒙恩,连淮维,夏瀚宇,姚明明,师铭泽,姜圣民,李之繁,管烁,易烊千玺,王俊凯,王源,丁程鑫,马嘉祺,张真源,敖子逸,李天泽,陈玺达,宋亚轩,刘耀文,贺俊霖,刘昊然,吴磊,郑凯,鹿晗,陈赫,李晨,邓超,包贝尔,王宝强,张翰,白敬亭,魏大勋,邓伦,汪苏泷,许嵩,徐良,张艺兴,道枝骏佑,片寄凉太,山崎贤人,黄子韬,吴世勋,边伯贤,朴灿烈,金钟大,金钟仁,都暻秀,金钟国,王嘉尔,刘宪华,杜海涛,沈腾,何炅,李维嘉,薛之谦,杨洋,华晨宇,纪凌尘,陈翔,车银尤,南赫柱,王力宏,胡先煦,马可,任嘉伦,李荣浩,艾芙杰尼,邢昭林,林更新,华少翌,黄晓明,韩寒,韩庚,韩磊,海岩,海清,海鸣威,胡军,胡海泉,胡歌,吴亦凡,张国立,唐国强,姜文,葛优,黄渤,陈坤,张一山,王传君,于和伟,刘欢,林志炫,徐峥,金志文,蒋昌建,品冠,张伟,杜飞,雷佳音,欧豪,高云翔,刘奕君,佟大为,郭京飞,张鲁一,王凯,霍建华,张钧甯,林子闳,马振桓,宋芸桦,邵雨薇,任言恺,欧阳娜娜,宇宙,连晨翔,瑞莎,昆凌,杜天皓,芮娜,苏笠汶,李玉玺,李威,明道,梁又琳,游智翔,寇世勋,雨婷,沈建宏,姚蜜,萧敬腾,向以丞,黄心娣,柯佳嬿,殷悦,邵崇柏,余晋,高以翔,黄郁婷,谢依霖,潘仪君,苏见信,张熙恩,石知田,柯宇纶,寇家瑞,陈威翰,柯家豪,康茵茵,康康,柯雅馨,王耀庆,王大陆,郭采洁,文雨非,蔡芷纭,陈奕,简远信,文淇,李凯馨,陈佩骐,黄维德,张轩睿,蔡颐榛,徐洁儿,王雅慧,陈柏融,王陈怡娴,邱胜翊,陈楚河,戴君竹,Teddy,锦荣,黄伟晋,张震,金士杰,吴中天,陈柏霖,郭雪芙,张庭,凤小岳,方妍心,林柏宏,欧阳妮妮,李鸿其,谢翔雅,杜妍,刘德凯,江语晨,安钧璨,李立群,白歆惠,陈薇,黄柏钧,许名杰,田中千绘,郑靓歆,那维勋,程席榆,纪欣妤,胡因梦,田丽,何海东,林佑威,陈怡真,陈盈燕,郭品超,黄文豪,小甜甜,吴若瑄,邱心志,林宥嘉,安心亚,韩忠羽,倪安东,修杰楷,常铖,禾浩辰,王轶玲,陈德修,倪齐民,易柏辰,丁文琪,李又麟,官鸿,洪小铃,王以纶,李程彬,屈中恒,王阳明,王棠云,葛蕾,郝劭文,邱昊奇,李天柱,狄志杰,陈亦飞,林韦君,温升豪,桂纶镁,张皓明,郑开元,黄少祺,陈敬宣,何恋慈,潘迎紫,阿本,方芳,张立昂,庄濠全,归亚蕾,王思懿,李毓芬,杜姸,张天霖,许玮甯,包小柏,关颖,苟芸慧,简廷芮,杨丽菁,陈庭妮,陈匡怡,魏蔓,张玉嬿,谢欣颖,陈博正,徐嘉苇,邓美恩,陈艾熙,郭书瑶,高凌风,谢和弦,李铨,徐贵樱,许富翔,张榕容,陈玺安,赖煜哲,金燕玲,亚里,连静雯,张国柱,许雅钧,白梓轩,杨世瀚,刘以豪,戴立忍,谢祖武,张孝全,夏若妍,唐国忠,陈乃荣,易恩,秦杨,王宥胜,蔡淑臻,高捷,小煜,刘乐妍,王维琳,刘瑞琪,张睿家,林逸欣,曾佩瑜,郭思辰,安娜,杜孟竹,吴可熙,陈汉典,黄瀞怡,蔡黄汝,叶全真,安晨芯,秦汉,纪亚文,吴心缇,陈尚泽,曲澔浚,李依瑾,陈艾琳,陈语安,纪培慧,立威廉,猪哥亮,杨青倩,陈建豪,是元介,刘冠毅,谢盈萱,郭鑫,孙克杰,张勋杰,张静懿,陈庭萱,俞小凡,詹子晴,郑暐达,风田,赖宗赢,曹华恩,张行,蔡旻佑,张景岚,江宏恩,孟庭丽,宋新妮,刘奕儿,王诗安,海狗,宋达民,吕孔维,蔡维泽,杨谨华,周凯文,卓毓彤,朱主爱,庹宗华,姚凤凤,刘国劭,孙兴,王心嫚,田家达,夏台凤,姚元浩,萧蔷,席惟伦,陈羽凡,陈昭荣,周子瑜,金超群,谢坤达,傅雷,姬天语,费翔,阿喜,周丹莉,李国超,沈彦廷,沈海蓉,曾子余,潘慧如,李志希,黄牛,黄丽玲,陶嫚曼,蓝心湄,纪言恺,刘俊纬,黄立行,方志友,吴思贤,宫以腾,包小松,徐佳莹,杨千霈,陈景炀,胡玮杰,龙隆,李运庆,关诗敏,刘泯廷,许韶洋,谢语恩,施羽,王家梁,谭艾珍,王彩桦,龚继安,艾伟,邓丽君,徐敏,钱韦杉,吴辰君,杨贵媚,安东尼,王灿,林若亚,Selina,张瑀希,周绍栋,刘汉强,古斌,阿雅,郭源元,贺一航,吕雪凤,唐治平,林美秀,莫允雯,李元太,游艾迪,安哲,张佩华,夏靖庭,唐德惠,侯彦西,李罗,周俐葳,陈冠霖,龚海铭,张芯瑜,许雅涵,刘芳芸,BY2,蔡健雅,李维维,李沛旭,许光汉,简嫚书,陈为民,李国毅,林妍柔,涂世旻,林予晞,杨升达,辰烨,李佳颖,周孝安,林秀君,秦风,邓九云,谢雅琳,舒子晨,翁卡特,夏语心,傅传杰,黄腾浩,胡睿儿,陈若萍,张雁名,阿信,李妍憬,翁滋蔓,大飞,江淑娜,蓝正龙,林筱筠,梁如瑄,阿Ken,盛鉴,赵擎,素珠,郑家榆,郑人硕,李康宜,林芯蕾,周丹薇,杨琪,安佑德,黄裕翔,周咏轩,蔡昌宪,钟欣凌,安程希,张立威,郎祖筠,刘谦,司徒颖霜,丁当,陈武康,喻虹渊,王者心,卫子云,迪丽热巴,成毅,陈学冬,安悦溪,姜妍,杨紫,郑爽,关晓彤,何明翰,徐梵溪,李沁,宋轶,乔欣,王鸥,古力娜扎,张馨予,麦迪娜,张铭恩,张檬,吴优,尹正,罗云熙,陈瑶,侯明昊,蒋欣,张云龙,杨烁,胡冰卿,靳东,牟星,袁姗姗,张彬彬,罗晋,秦俊杰,赵圆瑗,唐艺昕,李纯,梁振伦,吴倩,米露,王子文,TFBOYS,陈晓,张若昀,晁然,陈梦希,乔任梁,孙佳雨,李宏毅,林秋楠,茅子俊,唐娜,赖雨蒙,马思纯,于朦胧,徐璐,宋茜,杨菲洋,阚清子,孙耀琦,张萌,田倚凡,徐海乔,王妍之,戴向宇,刘芊含,熊梓淇,蔡文静,杨蓉,王珊,李心艾,何杜娟,王洋,井柏然,张雪迎,赵韩樱子,曹曦月,姜潮,孙怡,周冬雨,米勒,焦俊艳,杨子姗,逯恣祯,李溪芮,刘美含,刘萌萌,景甜,李一桐,杨旭文,梁洁,柳希龙,郑业成,李晟,陈小纭,黄礼格,林允,徐冬冬,宋威龙,袁冰妍,付辛博,沈梦辰,陈钰琪,方文强,鄂博,阎汶宣,牛骏峰,李小萌,白雪,张哲瀚,鞠婧祎,吕佳容,张浩,施诗,刘敏涛,王亭文,范世錡,海陆,马薇,邢菲,郭俊辰,孔垂楠,施予斐,穆婷婷,盛一伦,任天野,杨舒,刘庭羽,严屹宽,金晨,王晓晨,李砚,尹淇,韩栋,康杰,韩东君,于晓光,陈洁,张维娜,杨之楹,郑恺,蒋蕊泽,林源,张芷溪,苏青,蓝波,杨志刚,白百何,李佳奇,晏紫东,李健,高伟光,李进荣,张子枫,孙骁骁,米热,于震,宋小宝,刘芮麟,宋祖儿,李兰迪,康磊,叶祖新,于毅,蒋依依,李林娟,刘波,胡一天,石悦安鑫,娄清,刘娜萍,秦语,刘楚恬,王千源,阮圣文,孙雪宁,李倩,祝绪丹,母其弥雅,刘雅瑟,应昊茗,刘启恒,李梦,宋妍霏,金瀚,尹智玄,虞朗,袁志博,刘敏,任重,吴昊宸,孙艺洲,刘笑歌,毛林林,何泓姗,李郁瞳,袁泉,宋佳,马蜀君,尤靖茹,聂子皓,尚语贤,俞灏明,黄轩,吕卓燃,李飞,周奇奇,张予曦,刘文曲,刘秋实,李蓓蕾,潘之琳,梁超,泓萱,苗驰,卢宇静,童菲,王美人,刘颖伦,熊乃瑾,张碧晨,刘黛希,马秋子,刘烨,宋家腾,买红妹,祖峰,瑛子,江珊,杨雨婷,刘成瑞,李金铭,柯蓝,艾晓琪,刘馨棋,孙铱,白宇,杜淳,马春瑞,雨婷儿,杨帆,隋雨蒙,李颖,何晟铭,梁亦芸,陆子艺,张雨绮,于月仙,孙坚,岳丽娜,袁文康,梁浩,盛冠森,李春嫒,阳蕾,刘恩佑,杨梓墨,李子峰,吴谨言,张睿,孙茜,李茂,荣梓杉,陆妍淇,荣蓉,任帅,黄一琳,原雨,殷桃,辛芷蕾,翟天临,李欣聪,张晓谦,李雨轩,王珂,刘智扬,张楷依,于函冰,柴碧云,刘倩文,于小彤,尹铸胜,于笑,吉克隽逸,贾乃亮,童苡萱,宋奕星,刘雨欣,刘琳,姜鸿,安泳畅,王乐君,杨曦,于越,张晓晨,袁雨萱,马丽,林思意,陆昱霖,宋伊人,甘婷婷,刘天佐,李程,林静,李明德,林永健,冉旭,苏倩薇,叶峰,任柯诺,刘芸,叶青,马程程,卢杉,刘恬汝,丁婷婷,梁晶晶,李乃文,季晨,梁林琳,林江国,杀漠,李帅,张梦露,张峻宁,刘冠翔,刘丛丹,尹馨梓,路晨,米紫安,杨昆,李呈媛,徐申东,虞书欣,余心恬,梁爱琪,申奥,林千鹿,李昕岳,李萌萌,沈航,刘潺,穆乐恩,王彦霖,周雨彤,侍宣如,李嘉雯,王闯,李泽,余皑磊,苑子艺,赵予熙,李芯逸,刘钇彤,侯佩杉,侯梦瑶,姚奕辰,石安妮,施潮锦子,井星文,马晓伟,马灿灿,隋咏良,楼佳悦,陈思成,岳旸,马靓,陈都灵,李茜,卢星宇,李一情,何穗,李超,张炯敏,沙溢,王挺,SNH48,陆骏瑶,陈雅婷,李依伊 diff --git a/make_datasets/make_pix2pix_dataset.py b/make_datasets/make_pix2pix_dataset.py index c9ccb87..c87dc30 100644 --- a/make_datasets/make_pix2pix_dataset.py +++ b/make_datasets/make_pix2pix_dataset.py @@ -16,7 +16,7 @@ from models import runmodel,loadmodel import util.image_processing as impro -from util import util,mosaic,data +from util import degradater, util,mosaic,data opt.parser.add_argument('--datadir',type=str,default='../datasets/draw/face', help='') @@ -87,11 +87,11 @@ mask = mask_drawn if 'irregular' in opt.mod: mask_irr = impro.imread(irrpaths[random.randint(0,12000-1)],'gray') - mask_irr = data.random_transform_single(mask_irr, (img.shape[0],img.shape[1])) + mask_irr = data.random_transform_single_mask(mask_irr, (img.shape[0],img.shape[1])) mask = mask_irr if 'network' in opt.mod: mask_net = runmodel.get_ROI_position(img,net,opt,keepsize=True)[0] - if opt.use_gpu != -1: + if opt.gpu_id != -1: torch.cuda.empty_cache() if not opt.all_mosaic_area: mask_net = impro.find_mostlikely_ROI(mask_net) @@ -107,11 +107,11 @@ saveflag = True if opt.mod == ['drawn','irregular']: - x,y,size,area = impro.boundingSquare(mask_drawn, random.uniform(1.2,1.6)) + x,y,size,area = impro.boundingSquare(mask_drawn, random.uniform(1.1,1.6)) elif opt.mod == ['network','irregular']: - x,y,size,area = impro.boundingSquare(mask_net, random.uniform(1.2,1.6)) + x,y,size,area = impro.boundingSquare(mask_net, random.uniform(1.1,1.6)) else: - x,y,size,area = impro.boundingSquare(mask, random.uniform(1.2,1.6)) + x,y,size,area = impro.boundingSquare(mask, random.uniform(1.1,1.6)) if area < 1000: saveflag = False @@ -130,11 +130,15 @@ if saveflag: # add mosaic img_mosaic = mosaic.addmosaic_random(img, mask) - # random blur + # random degradater if random.random()>0.5: - Q = random.randint(1,15) - img = impro.dctblur(img,Q) - img_mosaic = impro.dctblur(img_mosaic,Q) + degradate_params = degradater.get_random_degenerate_params(mod='weaker_2') + img = degradater.degradate(img,degradate_params) + img_mosaic = degradater.degradate(img_mosaic,degradate_params) + # if random.random()>0.5: + # Q = random.randint(1,15) + # img = impro.dctblur(img,Q) + # img_mosaic = impro.dctblur(img_mosaic,Q) savecnt += 1 diff --git a/make_datasets/make_video_dataset.py b/make_datasets/make_video_dataset.py index b9ce6cb..84ef5fa 100644 --- a/make_datasets/make_video_dataset.py +++ b/make_datasets/make_video_dataset.py @@ -14,7 +14,7 @@ from models import runmodel,loadmodel import util.image_processing as impro -from util import util,mosaic,data,ffmpeg +from util import filt, util,mosaic,data,ffmpeg opt.parser.add_argument('--datadir',type=str,default='your video dir', help='') @@ -56,6 +56,7 @@ ffmpeg.video2image(videopath, opt.temp_dir+'/video2image/%05d.'+opt.tempimage_type,fps=1, start_time = util.second2stamp(cut_point*opt.interval),last_time = util.second2stamp(opt.time)) imagepaths = util.Traversal(opt.temp_dir+'/video2image') + imagepaths = sorted(imagepaths) cnt = 0 for i in range(opt.time): img = impro.imread(imagepaths[i]) @@ -92,30 +93,65 @@ imagepaths = util.Traversal(opt.temp_dir+'/video2image') imagepaths = sorted(imagepaths) imgs=[];masks=[] - mask_flag = False - + # mask_flag = False + # for imagepath in imagepaths: + # img = impro.imread(imagepath) + # mask = runmodel.get_ROI_position(img,net,opt,keepsize=True)[0] + # imgs.append(img) + # masks.append(mask) + # if not mask_flag: + # mask_avg = mask.astype(np.float64) + # mask_flag = True + # else: + # mask_avg += mask.astype(np.float64) + + # mask_avg = np.clip(mask_avg/len(imagepaths),0,255).astype('uint8') + # mask_avg = impro.mask_threshold(mask_avg,20,64) + # if not opt.all_mosaic_area: + # mask_avg = impro.find_mostlikely_ROI(mask_avg) + # x,y,size,area = impro.boundingSquare(mask_avg,Ex_mul=random.uniform(1.1,1.5)) + + # for i in range(len(imagepaths)): + # img = impro.resize(imgs[i][y-size:y+size,x-size:x+size],opt.outsize,interpolation=cv2.INTER_CUBIC) + # mask = impro.resize(masks[i][y-size:y+size,x-size:x+size],opt.outsize,interpolation=cv2.INTER_CUBIC) + # impro.imwrite(os.path.join(origindir,'%05d'%(i+1)+'.jpg'), img) + # impro.imwrite(os.path.join(maskdir,'%05d'%(i+1)+'.png'), mask) + ex_mul = random.uniform(1.2,1.7) + positions = [] for imagepath in imagepaths: img = impro.imread(imagepath) mask = runmodel.get_ROI_position(img,net,opt,keepsize=True)[0] imgs.append(img) masks.append(mask) - if not mask_flag: - mask_avg = mask.astype(np.float64) - mask_flag = True - else: - mask_avg += mask.astype(np.float64) - - mask_avg = np.clip(mask_avg/len(imagepaths),0,255).astype('uint8') - mask_avg = impro.mask_threshold(mask_avg,20,64) - if not opt.all_mosaic_area: - mask_avg = impro.find_mostlikely_ROI(mask_avg) - x,y,size,area = impro.boundingSquare(mask_avg,Ex_mul=random.uniform(1.1,1.5)) - - for i in range(len(imagepaths)): - img = impro.resize(imgs[i][y-size:y+size,x-size:x+size],opt.outsize,interpolation=cv2.INTER_CUBIC) + x,y,size,area = impro.boundingSquare(mask,Ex_mul=ex_mul) + positions.append([x,y,size]) + positions =np.array(positions) + for i in range(3):positions[:,i] = filt.medfilt(positions[:,i],opt.medfilt_num) + + for i,imagepath in enumerate(imagepaths): + x,y,size = positions[i][0],positions[i][1],positions[i][2] + tmp_cnt = i + while sizeopt.minsize//4: + # if not opt.all_mosaic_area: + # mask_avg = impro.find_mostlikely_ROI(mask_avg) + # x,y,size,area = impro.boundingSquare(mask_avg,Ex_mul=ex_mul) + # img = impro.resize(imgs[i][y-size:y+size,x-size:x+size],opt.outsize,interpolation=cv2.INTER_CUBIC) + # mask = impro.resize(masks[i][y-size:y+size,x-size:x+size],opt.outsize,interpolation=cv2.INTER_CUBIC) + # impro.imwrite(os.path.join(origindir,'%05d'%(i+1)+'.jpg'), img) + # impro.imwrite(os.path.join(maskdir,'%05d'%(i+1)+'.png'), mask) + result_cnt+=1 @@ -124,5 +160,5 @@ util.writelog(os.path.join(opt.savedir,'opt.txt'), videopath+'\n'+str(result_cnt)+'\n'+str(e)) video_cnt +=1 - if opt.use_gpu != -1: + if opt.gpu_id != '-1': torch.cuda.empty_cache() diff --git a/models/BVDNet.py b/models/BVDNet.py new file mode 100644 index 0000000..f9ca814 --- /dev/null +++ b/models/BVDNet.py @@ -0,0 +1,198 @@ +import torch +import torch.nn as nn +from .pix2pixHD_model import * +from .model_util import * +from models import model_util + +class UpBlock(nn.Module): + def __init__(self, in_channel, out_channel, kernel_size=3, padding=1): + super().__init__() + + self.convup = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False), + nn.ReflectionPad2d(padding), + # EqualConv2d(out_channel, out_channel, kernel_size, padding=padding), + SpectralNorm(nn.Conv2d(in_channel, out_channel, kernel_size)), + nn.LeakyReLU(0.2), + # Blur(out_channel), + ) + + def forward(self, input): + outup = self.convup(input) + return outup + +class Encoder2d(nn.Module): + def __init__(self, input_nc, ngf=64, n_downsampling=3, activation = nn.LeakyReLU(0.2)): + super(Encoder2d, self).__init__() + + model = [nn.ReflectionPad2d(3), SpectralNorm(nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0)), activation] + ### downsample + for i in range(n_downsampling): + mult = 2**i + model += [ nn.ReflectionPad2d(1), + SpectralNorm(nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=0)), + activation] + + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + +class Encoder3d(nn.Module): + def __init__(self, input_nc, ngf=64, n_downsampling=3, activation = nn.LeakyReLU(0.2)): + super(Encoder3d, self).__init__() + + model = [SpectralNorm(nn.Conv3d(input_nc, ngf, kernel_size=3, padding=1)), activation] + ### downsample + for i in range(n_downsampling): + mult = 2**i + model += [ SpectralNorm(nn.Conv3d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1)), + activation] + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + +class BVDNet(nn.Module): + def __init__(self, N=2, n_downsampling=3, n_blocks=4, input_nc=3, output_nc=3,activation=nn.LeakyReLU(0.2)): + super(BVDNet, self).__init__() + ngf = 64 + padding_type = 'reflect' + self.N = N + + ### encoder + self.encoder3d = Encoder3d(input_nc,64,n_downsampling,activation) + self.encoder2d = Encoder2d(input_nc,64,n_downsampling,activation) + + ### resnet blocks + self.blocks = [] + mult = 2**n_downsampling + for i in range(n_blocks): + self.blocks += [ResnetBlockSpectralNorm(ngf * mult, padding_type=padding_type, activation=activation)] + self.blocks = nn.Sequential(*self.blocks) + + ### decoder + self.decoder = [] + for i in range(n_downsampling): + mult = 2**(n_downsampling - i) + self.decoder += [UpBlock(ngf * mult, int(ngf * mult / 2))] + self.decoder += [nn.ReflectionPad2d(3), nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + self.decoder = nn.Sequential(*self.decoder) + self.limiter = nn.Tanh() + + def forward(self, stream, previous): + this_shortcut = stream[:,:,self.N] + stream = self.encoder3d(stream) + stream = stream.reshape(stream.size(0),stream.size(1),stream.size(3),stream.size(4)) + previous = self.encoder2d(previous) + x = stream + previous + x = self.blocks(x) + x = self.decoder(x) + x = x+this_shortcut + x = self.limiter(x) + return x + +def define_G(N=2, n_blocks=1, gpu_id='-1'): + netG = BVDNet(N = N, n_blocks=n_blocks) + netG = model_util.todevice(netG,gpu_id) + netG.apply(model_util.init_weights) + return netG + +################################Discriminator################################ +def define_D(input_nc=6, ndf=64, n_layers_D=1, use_sigmoid=False, num_D=3, gpu_id='-1'): + netD = MultiscaleDiscriminator(input_nc, ndf, n_layers_D, use_sigmoid, num_D) + netD = model_util.todevice(netD,gpu_id) + netD.apply(model_util.init_weights) + return netD + +class MultiscaleDiscriminator(nn.Module): + def __init__(self, input_nc, ndf=64, n_layers=3, use_sigmoid=False, num_D=3): + super(MultiscaleDiscriminator, self).__init__() + self.num_D = num_D + self.n_layers = n_layers + + for i in range(num_D): + netD = NLayerDiscriminator(input_nc, ndf, n_layers, use_sigmoid) + setattr(self, 'layer'+str(i), netD.model) + self.downsample = nn.AvgPool2d(3, stride=2, padding=[1, 1], count_include_pad=False) + + def singleD_forward(self, model, input): + return [model(input)] + + def forward(self, input): + num_D = self.num_D + result = [] + input_downsampled = input + for i in range(num_D): + model = getattr(self, 'layer'+str(num_D-1-i)) + result.append(self.singleD_forward(model, input_downsampled)) + if i != (num_D-1): + input_downsampled = self.downsample(input_downsampled) + return result + +# Defines the PatchGAN discriminator with the specified arguments. +class NLayerDiscriminator(nn.Module): + def __init__(self, input_nc, ndf=64, n_layers=3, use_sigmoid=False): + super(NLayerDiscriminator, self).__init__() + self.n_layers = n_layers + + kw = 4 + padw = int(np.ceil((kw-1.0)/2)) + sequence = [[nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2)]] + + nf = ndf + for n in range(1, n_layers): + nf_prev = nf + nf = min(nf * 2, 512) + sequence += [[ + SpectralNorm(nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=2, padding=padw)), + nn.LeakyReLU(0.2) + ]] + + nf_prev = nf + nf = min(nf * 2, 512) + sequence += [[ + SpectralNorm(nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=1, padding=padw)), + nn.LeakyReLU(0.2) + ]] + + sequence += [[nn.Conv2d(nf, 1, kernel_size=kw, stride=1, padding=padw)]] + + if use_sigmoid: + sequence += [[nn.Sigmoid()]] + + sequence_stream = [] + for n in range(len(sequence)): + sequence_stream += sequence[n] + self.model = nn.Sequential(*sequence_stream) + + def forward(self, input): + return self.model(input) + +class GANLoss(nn.Module): + def __init__(self, mode='D'): + super(GANLoss, self).__init__() + if mode == 'D': + self.lossf = model_util.HingeLossD() + elif mode == 'G': + self.lossf = model_util.HingeLossG() + self.mode = mode + + def forward(self, dis_fake = None, dis_real = None): + if isinstance(dis_fake, list): + if self.mode == 'D': + loss = 0 + for i in range(len(dis_fake)): + loss += self.lossf(dis_fake[i][-1],dis_real[i][-1]) + elif self.mode =='G': + loss = 0 + weight = 2**len(dis_fake) + for i in range(len(dis_fake)): + weight = weight/2 + loss += weight*self.lossf(dis_fake[i][-1]) + return loss + else: + if self.mode == 'D': + return self.lossf(dis_fake[-1],dis_real[-1]) + elif self.mode =='G': + return self.lossf(dis_fake[-1]) diff --git a/models/BiSeNet_model.py b/models/BiSeNet_model.py index b58ea5b..3675141 100644 --- a/models/BiSeNet_model.py +++ b/models/BiSeNet_model.py @@ -2,7 +2,7 @@ import torch.nn as nn import torch import torch.nn.functional as F -from . import components +from . import model_util import warnings warnings.filterwarnings(action='ignore') @@ -43,7 +43,7 @@ def forward(self, output, target): class resnet18(torch.nn.Module): def __init__(self, pretrained=True): super().__init__() - self.features = components.resnet18(pretrained=pretrained) + self.features = model_util.resnet18(pretrained=pretrained) self.conv1 = self.features.conv1 self.bn1 = self.features.bn1 self.relu = self.features.relu @@ -70,7 +70,7 @@ def forward(self, input): class resnet101(torch.nn.Module): def __init__(self, pretrained=True): super().__init__() - self.features = components.resnet101(pretrained=pretrained) + self.features = model_util.resnet101(pretrained=pretrained) self.conv1 = self.features.conv1 self.bn1 = self.features.bn1 self.relu = self.features.relu diff --git a/models/components.py b/models/components.py deleted file mode 100644 index 59cb333..0000000 --- a/models/components.py +++ /dev/null @@ -1,234 +0,0 @@ -import torch.nn as nn -import torch.utils.model_zoo as model_zoo - - -__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', - 'resnet152'] - - -model_urls = { - 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', - 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', - 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', - 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', - 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', -} - - -def conv3x3(in_planes, out_planes, stride=1): - """3x3 convolution with padding""" - return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, - padding=1, bias=False) - - -def conv1x1(in_planes, out_planes, stride=1): - """1x1 convolution""" - return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) - - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=None): - super(BasicBlock, self).__init__() - if norm_layer is None: - norm_layer = nn.BatchNorm2d - # Both self.conv1 and self.downsample layers downsample the input when stride != 1 - self.conv1 = conv3x3(inplanes, planes, stride) - self.bn1 = norm_layer(planes) - self.relu = nn.ReLU(inplace=True) - self.conv2 = conv3x3(planes, planes) - self.bn2 = norm_layer(planes) - self.downsample = downsample - self.stride = stride - - def forward(self, x): - identity = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - - out = self.conv2(out) - out = self.bn2(out) - - if self.downsample is not None: - identity = self.downsample(x) - - out += identity - out = self.relu(out) - - return out - - -class Bottleneck(nn.Module): - expansion = 4 - - def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=None): - super(Bottleneck, self).__init__() - if norm_layer is None: - norm_layer = nn.BatchNorm2d - # Both self.conv2 and self.downsample layers downsample the input when stride != 1 - self.conv1 = conv1x1(inplanes, planes) - self.bn1 = norm_layer(planes) - self.conv2 = conv3x3(planes, planes, stride) - self.bn2 = norm_layer(planes) - self.conv3 = conv1x1(planes, planes * self.expansion) - self.bn3 = norm_layer(planes * self.expansion) - self.relu = nn.ReLU(inplace=True) - self.downsample = downsample - self.stride = stride - - def forward(self, x): - identity = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - - out = self.conv2(out) - out = self.bn2(out) - out = self.relu(out) - - out = self.conv3(out) - out = self.bn3(out) - - if self.downsample is not None: - identity = self.downsample(x) - - out += identity - out = self.relu(out) - - return out - - -class ResNet(nn.Module): - - def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, norm_layer=None): - super(ResNet, self).__init__() - if norm_layer is None: - norm_layer = nn.BatchNorm2d - self.inplanes = 64 - self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, - bias=False) - self.bn1 = norm_layer(64) - self.relu = nn.ReLU(inplace=True) - self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer) - self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer) - self.layer3 = self._make_layer(block, 256, layers[2], stride=2, norm_layer=norm_layer) - self.layer4 = self._make_layer(block, 512, layers[3], stride=2, norm_layer=norm_layer) - self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) - self.fc = nn.Linear(512 * block.expansion, num_classes) - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') - elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): - nn.init.constant_(m.weight, 1) - nn.init.constant_(m.bias, 0) - - # Zero-initialize the last BN in each residual branch, - # so that the residual branch starts with zeros, and each residual block behaves like an identity. - # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 - if zero_init_residual: - for m in self.modules(): - if isinstance(m, Bottleneck): - nn.init.constant_(m.bn3.weight, 0) - elif isinstance(m, BasicBlock): - nn.init.constant_(m.bn2.weight, 0) - - def _make_layer(self, block, planes, blocks, stride=1, norm_layer=None): - if norm_layer is None: - norm_layer = nn.BatchNorm2d - downsample = None - if stride != 1 or self.inplanes != planes * block.expansion: - downsample = nn.Sequential( - conv1x1(self.inplanes, planes * block.expansion, stride), - norm_layer(planes * block.expansion), - ) - - layers = [] - layers.append(block(self.inplanes, planes, stride, downsample, norm_layer)) - self.inplanes = planes * block.expansion - for _ in range(1, blocks): - layers.append(block(self.inplanes, planes, norm_layer=norm_layer)) - - return nn.Sequential(*layers) - - def forward(self, x): - x = self.conv1(x) - x = self.bn1(x) - x = self.relu(x) - x = self.maxpool(x) - - x = self.layer1(x) - x = self.layer2(x) - x = self.layer3(x) - x = self.layer4(x) - - x = self.avgpool(x) - x = x.view(x.size(0), -1) - x = self.fc(x) - - return x - - -def resnet18(pretrained=False, **kwargs): - """Constructs a ResNet-18 model. - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet18'])) - return model - - -def resnet34(pretrained=False, **kwargs): - """Constructs a ResNet-34 model. - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet34'])) - return model - - -def resnet50(pretrained=False, **kwargs): - """Constructs a ResNet-50 model. - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet50'])) - return model - - -def resnet101(pretrained=False, **kwargs): - """Constructs a ResNet-101 model. - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet101'])) - return model - - -def resnet152(pretrained=False, **kwargs): - """Constructs a ResNet-152 model. - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet152'])) - return model \ No newline at end of file diff --git a/models/loadmodel.py b/models/loadmodel.py index 974064a..7d9b391 100755 --- a/models/loadmodel.py +++ b/models/loadmodel.py @@ -1,49 +1,35 @@ import torch -from .pix2pix_model import define_G -from .pix2pixHD_model import define_G as define_G_HD -from .unet_model import UNet -from .video_model import MosaicNet -from .videoHD_model import MosaicNet as MosaicNet_HD +from . import model_util +from .pix2pix_model import define_G as pix2pix_G +from .pix2pixHD_model import define_G as pix2pixHD_G +# from .video_model import MosaicNet +# from .videoHD_model import MosaicNet as MosaicNet_HD from .BiSeNet_model import BiSeNet +from .BVDNet import define_G as video_G def show_paramsnumber(net,netname='net'): parameters = sum(param.numel() for param in net.parameters()) parameters = round(parameters/1e6,2) print(netname+' parameters: '+str(parameters)+'M') -def __patch_instance_norm_state_dict(state_dict, module, keys, i=0): - """Fix InstanceNorm checkpoints incompatibility (prior to 0.4)""" - key = keys[i] - if i + 1 == len(keys): # at the end, pointing to a parameter/buffer - if module.__class__.__name__.startswith('InstanceNorm') and \ - (key == 'running_mean' or key == 'running_var'): - if getattr(module, key) is None: - state_dict.pop('.'.join(keys)) - if module.__class__.__name__.startswith('InstanceNorm') and \ - (key == 'num_batches_tracked'): - state_dict.pop('.'.join(keys)) - else: - __patch_instance_norm_state_dict(state_dict, getattr(module, key), keys, i + 1) - def pix2pix(opt): # print(opt.model_path,opt.netG) if opt.netG == 'HD': - netG = define_G_HD(3, 3, 64, 'global' ,4) + netG = pix2pixHD_G(3, 3, 64, 'global' ,4) else: - netG = define_G(3, 3, 64, opt.netG, norm='batch',use_dropout=True, init_type='normal', gpu_ids=[]) + netG = pix2pix_G(3, 3, 64, opt.netG, norm='batch',use_dropout=True, init_type='normal', gpu_ids=[]) show_paramsnumber(netG,'netG') netG.load_state_dict(torch.load(opt.model_path)) + netG = model_util.todevice(netG,opt.gpu_id) netG.eval() - if opt.use_gpu != -1: - netG.cuda() return netG def style(opt): if opt.edges: - netG = define_G(1, 3, 64, 'resnet_9blocks', norm='instance',use_dropout=True, init_type='normal', gpu_ids=[]) + netG = pix2pix_G(1, 3, 64, 'resnet_9blocks', norm='instance',use_dropout=True, init_type='normal', gpu_ids=[]) else: - netG = define_G(3, 3, 64, 'resnet_9blocks', norm='instance',use_dropout=False, init_type='normal', gpu_ids=[]) + netG = pix2pix_G(3, 3, 64, 'resnet_9blocks', norm='instance',use_dropout=False, init_type='normal', gpu_ids=[]) #in other to load old pretrain model #https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/models/base_model.py @@ -57,23 +43,19 @@ def style(opt): # patch InstanceNorm checkpoints prior to 0.4 for key in list(state_dict.keys()): # need to copy keys here because we mutate in loop - __patch_instance_norm_state_dict(state_dict, netG, key.split('.')) + model_util.patch_instance_norm_state_dict(state_dict, netG, key.split('.')) netG.load_state_dict(state_dict) - if opt.use_gpu != -1: - netG.cuda() + netG = model_util.todevice(netG,opt.gpu_id) + netG.eval() return netG def video(opt): - if 'HD' in opt.model_path: - netG = MosaicNet_HD(3*25+1, 3, norm='instance') - else: - netG = MosaicNet(3*25+1, 3,norm = 'batch') + netG = video_G(N=2,n_blocks=4,gpu_id=opt.gpu_id) show_paramsnumber(netG,'netG') netG.load_state_dict(torch.load(opt.model_path)) + netG = model_util.todevice(netG,opt.gpu_id) netG.eval() - if opt.use_gpu != -1: - netG.cuda() return netG def bisenet(opt,type='roi'): @@ -86,7 +68,6 @@ def bisenet(opt,type='roi'): net.load_state_dict(torch.load(opt.model_path)) elif type == 'mosaic': net.load_state_dict(torch.load(opt.mosaic_position_model_path)) + net = model_util.todevice(net,opt.gpu_id) net.eval() - if opt.use_gpu != -1: - net.cuda() return net diff --git a/models/model_util.py b/models/model_util.py new file mode 100644 index 0000000..2aa7f9e --- /dev/null +++ b/models/model_util.py @@ -0,0 +1,469 @@ +import functools +from math import exp + +import torch +import torch.nn as nn +from torch.nn import init +from torch.autograd import Variable +import torch.nn.functional as F +import torch.nn.utils.spectral_norm as SpectralNorm +from torchvision import models +import torch.utils.model_zoo as model_zoo + +################################## IO ################################## +def save(net,path,gpu_id): + if isinstance(net, nn.DataParallel): + torch.save(net.module.cpu().state_dict(),path) + else: + torch.save(net.cpu().state_dict(),path) + if gpu_id != '-1': + net.cuda() + +def todevice(net,gpu_id): + if gpu_id != '-1' and len(gpu_id) == 1: + net.cuda() + elif gpu_id != '-1' and len(gpu_id) > 1: + net = nn.DataParallel(net) + net.cuda() + return net + +# patch InstanceNorm checkpoints prior to 0.4 +def patch_instance_norm_state_dict(state_dict, module, keys, i=0): + """Fix InstanceNorm checkpoints incompatibility (prior to 0.4)""" + key = keys[i] + if i + 1 == len(keys): # at the end, pointing to a parameter/buffer + if module.__class__.__name__.startswith('InstanceNorm') and \ + (key == 'running_mean' or key == 'running_var'): + if getattr(module, key) is None: + state_dict.pop('.'.join(keys)) + if module.__class__.__name__.startswith('InstanceNorm') and \ + (key == 'num_batches_tracked'): + state_dict.pop('.'.join(keys)) + else: + patch_instance_norm_state_dict(state_dict, getattr(module, key), keys, i + 1) + +################################## initialization ################################## +def get_norm_layer(norm_type='instance',mod = '2d'): + if norm_type == 'batch': + if mod == '2d': + norm_layer = functools.partial(nn.BatchNorm2d, affine=True) + elif mod == '3d': + norm_layer = functools.partial(nn.BatchNorm3d, affine=True) + elif norm_type == 'instance': + if mod == '2d': + norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=True) + elif mod =='3d': + norm_layer = functools.partial(nn.InstanceNorm3d, affine=False, track_running_stats=True) + elif norm_type == 'none': + norm_layer = None + else: + raise NotImplementedError('normalization layer [%s] is not found' % norm_type) + + return norm_layer + +def init_weights(net, init_type='normal', gain=0.02): + def init_func(m): + classname = m.__class__.__name__ + if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1): + if init_type == 'normal': + init.normal_(m.weight.data, 0.0, gain) + elif init_type == 'xavier': + init.xavier_normal_(m.weight.data, gain=gain) + elif init_type == 'kaiming': + init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + init.orthogonal_(m.weight.data, gain=gain) + else: + raise NotImplementedError('initialization method [%s] is not implemented' % init_type) + if hasattr(m, 'bias') and m.bias is not None: + init.constant_(m.bias.data, 0.0) + elif classname.find('BatchNorm2d') != -1: + init.normal_(m.weight.data, 1.0, gain) + init.constant_(m.bias.data, 0.0) + + # print('initialize network with %s' % init_type) + net.apply(init_func) + +################################## Network structure ################################## +################################## ResnetBlock ################################## +class ResnetBlockSpectralNorm(nn.Module): + def __init__(self, dim, padding_type, activation=nn.LeakyReLU(0.2), use_dropout=False): + super(ResnetBlockSpectralNorm, self).__init__() + self.conv_block = self.build_conv_block(dim, padding_type, activation, use_dropout) + + def build_conv_block(self, dim, padding_type, activation, use_dropout): + conv_block = [] + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + + conv_block += [SpectralNorm(nn.Conv2d(dim, dim, kernel_size=3, padding=p)), + activation] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + conv_block += [SpectralNorm(nn.Conv2d(dim, dim, kernel_size=3, padding=p))] + + return nn.Sequential(*conv_block) + + def forward(self, x): + out = x + self.conv_block(x) + return out + +################################## Resnet ################################## +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + +class Bottleneck(nn.Module): + expansion = 4 + def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, planes) + self.bn1 = norm_layer(planes) + self.conv2 = conv3x3(planes, planes, stride) + self.bn2 = norm_layer(planes) + self.conv3 = conv1x1(planes, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, norm_layer=None): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self.inplanes = 64 + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, norm_layer=norm_layer) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, norm_layer=norm_layer) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, norm_layer=None): + if norm_layer is None: + norm_layer = nn.BatchNorm2d + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x + +def resnet18(pretrained=False, **kwargs): + """Constructs a ResNet-18 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + if pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['resnet18'])) + return model + +def resnet101(pretrained=False, **kwargs): + """Constructs a ResNet-101 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) + if pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['resnet101'])) + return model + +################################## Loss function ################################## +class HingeLossD(nn.Module): + def __init__(self): + super(HingeLossD, self).__init__() + + def forward(self, dis_fake, dis_real): + loss_real = torch.mean(F.relu(1. - dis_real)) + loss_fake = torch.mean(F.relu(1. + dis_fake)) + return loss_real + loss_fake + +class HingeLossG(nn.Module): + def __init__(self): + super(HingeLossG, self).__init__() + + def forward(self, dis_fake): + loss_fake = -torch.mean(dis_fake) + return loss_fake + +class VGGLoss(nn.Module): + def __init__(self, gpu_id): + super(VGGLoss, self).__init__() + + self.vgg = Vgg19() + if gpu_id != '-1' and len(gpu_id) == 1: + self.vgg.cuda() + elif gpu_id != '-1' and len(gpu_id) > 1: + self.vgg = nn.DataParallel(self.vgg) + self.vgg.cuda() + + self.criterion = nn.MSELoss() + self.weights = [1.0/32, 1.0/16, 1.0/8, 1.0/4, 1.0] + + def forward(self, x, y): + x_vgg, y_vgg = self.vgg(x), self.vgg(y) + loss = 0 + for i in range(len(x_vgg)): + loss += self.weights[i] * self.criterion(x_vgg[i], y_vgg[i].detach()) + return loss + +class Vgg19(torch.nn.Module): + def __init__(self, requires_grad=False): + super(Vgg19, self).__init__() + vgg_pretrained_features = models.vgg19(pretrained=True).features + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + for x in range(2): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(2, 7): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(7, 12): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(12, 21): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(21, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + h_relu1 = self.slice1(X) + h_relu2 = self.slice2(h_relu1) + h_relu3 = self.slice3(h_relu2) + h_relu4 = self.slice4(h_relu3) + h_relu5 = self.slice5(h_relu4) + out = [h_relu1, h_relu2, h_relu3, h_relu4, h_relu5] + return out + +################################## Evaluation ################################## +'''https://github.com/Po-Hsun-Su/pytorch-ssim + +img1 = Variable(torch.rand(1, 1, 256, 256)) +img2 = Variable(torch.rand(1, 1, 256, 256)) + +if torch.cuda.is_available(): + img1 = img1.cuda() + img2 = img2.cuda() + +print(pytorch_ssim.ssim(img1, img2)) + +ssim_loss = pytorch_ssim.SSIM(window_size = 11) + +print(ssim_loss(img1, img2)) +''' + +def gaussian(window_size, sigma): + gauss = torch.Tensor([exp(-(x - window_size//2)**2/float(2*sigma**2)) for x in range(window_size)]) + return gauss/gauss.sum() + +def create_window(window_size, channel): + _1D_window = gaussian(window_size, 1.5).unsqueeze(1) + _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0) + window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous()) + return window + +def _ssim(img1, img2, window, window_size, channel, size_average = True): + mu1 = F.conv2d(img1, window, padding = window_size//2, groups = channel) + mu2 = F.conv2d(img2, window, padding = window_size//2, groups = channel) + + mu1_sq = mu1.pow(2) + mu2_sq = mu2.pow(2) + mu1_mu2 = mu1*mu2 + + sigma1_sq = F.conv2d(img1*img1, window, padding = window_size//2, groups = channel) - mu1_sq + sigma2_sq = F.conv2d(img2*img2, window, padding = window_size//2, groups = channel) - mu2_sq + sigma12 = F.conv2d(img1*img2, window, padding = window_size//2, groups = channel) - mu1_mu2 + + C1 = 0.01**2 + C2 = 0.03**2 + + ssim_map = ((2*mu1_mu2 + C1)*(2*sigma12 + C2))/((mu1_sq + mu2_sq + C1)*(sigma1_sq + sigma2_sq + C2)) + + if size_average: + return ssim_map.mean() + else: + return ssim_map.mean(1).mean(1).mean(1) + +class SSIM(torch.nn.Module): + def __init__(self, window_size = 11, size_average = True): + super(SSIM, self).__init__() + self.window_size = window_size + self.size_average = size_average + self.channel = 1 + self.window = create_window(window_size, self.channel) + + def forward(self, img1, img2): + (_, channel, _, _) = img1.size() + + if channel == self.channel and self.window.data.type() == img1.data.type(): + window = self.window + else: + window = create_window(self.window_size, channel) + + if img1.is_cuda: + window = window.cuda(img1.get_device()) + window = window.type_as(img1) + + self.window = window + self.channel = channel + + + return _ssim(img1, img2, window, self.window_size, channel, self.size_average) + +def ssim(img1, img2, window_size = 11, size_average = True): + (_, channel, _, _) = img1.size() + window = create_window(window_size, channel) + + if img1.is_cuda: + window = window.cuda(img1.get_device()) + window = window.type_as(img1) + + return _ssim(img1, img2, window, window_size, channel, size_average) diff --git a/models/runmodel.py b/models/runmodel.py index 2bdc88d..3e97fee 100755 --- a/models/runmodel.py +++ b/models/runmodel.py @@ -7,11 +7,11 @@ import torch import numpy as np -def run_segment(img,net,size = 360,use_gpu = 0): +def run_segment(img,net,size = 360,gpu_id = '-1'): img = impro.resize(img,size) - img = data.im2tensor(img,use_gpu = use_gpu, bgr2rgb = False,use_transform = False , is0_1 = True) + img = data.im2tensor(img,gpu_id = gpu_id, bgr2rgb = False, is0_1 = True) mask = net(img) - mask = data.tensor2im(mask, gray=True,rgb2bgr = False, is0_1 = True) + mask = data.tensor2im(mask, gray=True, is0_1 = True) return mask def run_pix2pix(img,net,opt): @@ -19,7 +19,7 @@ def run_pix2pix(img,net,opt): img = impro.resize(img,512) else: img = impro.resize(img,128) - img = data.im2tensor(img,use_gpu=opt.use_gpu) + img = data.im2tensor(img,gpu_id=opt.gpu_id) img_fake = net(img) img_fake = data.tensor2im(img_fake) return img_fake @@ -50,18 +50,18 @@ def run_styletransfer(opt, net, img): else: canny_low = opt.canny-int(opt.canny/2) canny_high = opt.canny+int(opt.canny/2) - img = cv2.Canny(img,opt.canny-50,opt.canny+50) + img = cv2.Canny(img,canny_low,canny_high) if opt.only_edges: return img - img = data.im2tensor(img,use_gpu=opt.use_gpu,gray=True,use_transform = False,is0_1 = False) + img = data.im2tensor(img,gpu_id=opt.gpu_id,gray=True) else: - img = data.im2tensor(img,use_gpu=opt.use_gpu,gray=False,use_transform = True) + img = data.im2tensor(img,gpu_id=opt.gpu_id) img = net(img) img = data.tensor2im(img) return img def get_ROI_position(img,net,opt,keepsize=True): - mask = run_segment(img,net,size=360,use_gpu = opt.use_gpu) + mask = run_segment(img,net,size=360,gpu_id = opt.gpu_id) mask = impro.mask_threshold(mask,opt.mask_extend,opt.mask_threshold) if keepsize: mask = impro.resize_like(mask, img) @@ -70,7 +70,7 @@ def get_ROI_position(img,net,opt,keepsize=True): def get_mosaic_position(img_origin,net_mosaic_pos,opt): h,w = img_origin.shape[:2] - mask = run_segment(img_origin,net_mosaic_pos,size=360,use_gpu = opt.use_gpu) + mask = run_segment(img_origin,net_mosaic_pos,size=360,gpu_id = opt.gpu_id) # mask_1 = mask.copy() mask = impro.mask_threshold(mask,ex_mun=int(min(h,w)/20),threshold=opt.mask_threshold) if not opt.all_mosaic_area: diff --git a/models/videoHD_model.py b/models/videoHD_model.py deleted file mode 100644 index 20e901f..0000000 --- a/models/videoHD_model.py +++ /dev/null @@ -1,173 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from .pix2pixHD_model import * - - -class encoder_2d(nn.Module): - def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, - padding_type='reflect'): - assert(n_blocks >= 0) - super(encoder_2d, self).__init__() - activation = nn.ReLU(True) - - model = [nn.ReflectionPad2d(3), nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0), norm_layer(ngf), activation] - ### downsample - for i in range(n_downsampling): - mult = 2**i - model += [nn.ReflectionPad2d(1),nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=0), - norm_layer(ngf * mult * 2), activation] - - self.model = nn.Sequential(*model) - def forward(self, input): - return self.model(input) - -class decoder_2d(nn.Module): - def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, - padding_type='reflect'): - assert(n_blocks >= 0) - super(decoder_2d, self).__init__() - activation = nn.ReLU(True) - - model = [] - - ### resnet blocks - mult = 2**n_downsampling - for i in range(n_blocks): - model += [ResnetBlock(ngf * mult, padding_type=padding_type, activation=activation, norm_layer=norm_layer)] - - ### upsample - for i in range(n_downsampling): - mult = 2**(n_downsampling - i) - - # model += [ nn.Upsample(scale_factor = 2, mode='nearest'), - # nn.ReflectionPad2d(1), - # nn.Conv2d(ngf * mult, int(ngf * mult / 2),kernel_size=3, stride=1, padding=0), - # norm_layer(int(ngf * mult / 2)), - # nn.ReLU(True)] - model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), kernel_size=3, stride=2, padding=1, output_padding=1), - norm_layer(int(ngf * mult / 2)), activation] - model += [nn.ReflectionPad2d(3), nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0), nn.Tanh()] - self.model = nn.Sequential(*model) - - def forward(self, input): - return self.model(input) - - -class conv_3d(nn.Module): - def __init__(self,inchannel,outchannel,kernel_size=3,stride=2,padding=1,norm_layer_3d=nn.BatchNorm3d,use_bias=True): - super(conv_3d, self).__init__() - self.conv = nn.Sequential( - nn.Conv3d(inchannel, outchannel, kernel_size=kernel_size, stride=stride, padding=padding, bias=use_bias), - norm_layer_3d(outchannel), - nn.ReLU(inplace=True), - ) - - def forward(self, x): - x = self.conv(x) - return x - -class conv_2d(nn.Module): - def __init__(self,inchannel,outchannel,kernel_size=3,stride=1,padding=1,norm_layer_2d=nn.BatchNorm2d,use_bias=True): - super(conv_2d, self).__init__() - self.conv = nn.Sequential( - nn.ReflectionPad2d(padding), - nn.Conv2d(inchannel, outchannel, kernel_size=kernel_size, stride=stride, padding=0, bias=use_bias), - norm_layer_2d(outchannel), - nn.ReLU(inplace=True), - ) - - def forward(self, x): - x = self.conv(x) - return x - - -class encoder_3d(nn.Module): - def __init__(self,in_channel,norm_layer_2d,norm_layer_3d,use_bias): - super(encoder_3d, self).__init__() - self.inconv = conv_3d(1, 64, 7, 2, 3,norm_layer_3d,use_bias) - self.down1 = conv_3d(64, 128, 3, 2, 1,norm_layer_3d,use_bias) - self.down2 = conv_3d(128, 256, 3, 2, 1,norm_layer_3d,use_bias) - self.down3 = conv_3d(256, 512, 3, 2, 1,norm_layer_3d,use_bias) - self.down4 = conv_3d(512, 1024, 3, 1, 1,norm_layer_3d,use_bias) - self.pool = nn.AvgPool3d((5,1,1)) - # self.conver2d = nn.Sequential( - # nn.Conv2d(256*int(in_channel/4), 256, kernel_size=3, stride=1, padding=1, bias=use_bias), - # norm_layer_2d(256), - # nn.ReLU(inplace=True), - # ) - - - def forward(self, x): - - x = x.view(x.size(0),1,x.size(1),x.size(2),x.size(3)) - x = self.inconv(x) - x = self.down1(x) - x = self.down2(x) - x = self.down3(x) - x = self.down4(x) - #print(x.size()) - x = self.pool(x) - #print(x.size()) - # torch.Size([1, 1024, 16, 16]) - # torch.Size([1, 512, 5, 16, 16]) - - - x = x.view(x.size(0),x.size(1),x.size(3),x.size(4)) - - # x = self.conver2d(x) - - return x - - # def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, - # padding_type='reflect') - -class ALL(nn.Module): - def __init__(self, in_channel, out_channel,norm_layer_2d,norm_layer_3d,use_bias): - super(ALL, self).__init__() - - self.encoder_2d = encoder_2d(4,3,64,4,norm_layer=norm_layer_2d,padding_type='reflect') - self.encoder_3d = encoder_3d(in_channel,norm_layer_2d,norm_layer_3d,use_bias) - self.decoder_2d = decoder_2d(4,3,64,4,norm_layer=norm_layer_2d,padding_type='reflect') - # self.shortcut_cov = conv_2d(3,64,7,1,3,norm_layer_2d,use_bias) - self.merge1 = conv_2d(2048,1024,3,1,1,norm_layer_2d,use_bias) - # self.merge2 = nn.Sequential( - # conv_2d(128,64,3,1,1,norm_layer_2d,use_bias), - # nn.ReflectionPad2d(3), - # nn.Conv2d(64, out_channel, kernel_size=7, padding=0), - # nn.Tanh() - # ) - - def forward(self, x): - - N = int((x.size()[1])/3) - x_2d = torch.cat((x[:,int((N-1)/2)*3:(int((N-1)/2)+1)*3,:,:], x[:,N-1:N,:,:]), 1) - #shortcut_2d = x[:,int((N-1)/2)*3:(int((N-1)/2)+1)*3,:,:] - - x_2d = self.encoder_2d(x_2d) - x_3d = self.encoder_3d(x) - #x = x_2d + x_3d - x = torch.cat((x_2d,x_3d),1) - x = self.merge1(x) - - x = self.decoder_2d(x) - #shortcut_2d = self.shortcut_cov(shortcut_2d) - #x = torch.cat((x,shortcut_2d),1) - #x = self.merge2(x) - - return x - -def MosaicNet(in_channel, out_channel, norm='batch'): - - if norm == 'batch': - # norm_layer_2d = nn.BatchNorm2d - # norm_layer_3d = nn.BatchNorm3d - norm_layer_2d = functools.partial(nn.BatchNorm2d, affine=True, track_running_stats=True) - norm_layer_3d = functools.partial(nn.BatchNorm3d, affine=True, track_running_stats=True) - use_bias = False - elif norm == 'instance': - norm_layer_2d = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False) - norm_layer_3d = functools.partial(nn.InstanceNorm3d, affine=False, track_running_stats=False) - use_bias = True - - return ALL(in_channel, out_channel, norm_layer_2d, norm_layer_3d, use_bias) diff --git a/models/video_model.py b/models/video_model.py deleted file mode 100644 index 4a095c6..0000000 --- a/models/video_model.py +++ /dev/null @@ -1,216 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from .pix2pix_model import * - - -class encoder_2d(nn.Module): - """Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations. - - We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style) - """ - - def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'): - """Construct a Resnet-based generator - - Parameters: - input_nc (int) -- the number of channels in input images - output_nc (int) -- the number of channels in output images - ngf (int) -- the number of filters in the last conv layer - norm_layer -- normalization layer - use_dropout (bool) -- if use dropout layers - n_blocks (int) -- the number of ResNet blocks - padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero - """ - assert(n_blocks >= 0) - super(encoder_2d, self).__init__() - if type(norm_layer) == functools.partial: - use_bias = norm_layer.func == nn.InstanceNorm2d - else: - use_bias = norm_layer == nn.InstanceNorm2d - - model = [nn.ReflectionPad2d(3), - nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias), - norm_layer(ngf), - nn.ReLU(True)] - - n_downsampling = 2 - for i in range(n_downsampling): # add downsampling layers - mult = 2 ** i - model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias), - norm_layer(ngf * mult * 2), - nn.ReLU(True)] - #torch.Size([1, 256, 32, 32]) - - self.model = nn.Sequential(*model) - - def forward(self, input): - """Standard forward""" - return self.model(input) - - -class decoder_2d(nn.Module): - """Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations. - - We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style) - """ - - def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'): - """Construct a Resnet-based generator - - Parameters: - input_nc (int) -- the number of channels in input images - output_nc (int) -- the number of channels in output images - ngf (int) -- the number of filters in the last conv layer - norm_layer -- normalization layer - use_dropout (bool) -- if use dropout layers - n_blocks (int) -- the number of ResNet blocks - padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero - """ - super(decoder_2d, self).__init__() - if type(norm_layer) == functools.partial: - use_bias = norm_layer.func == nn.InstanceNorm2d - else: - use_bias = norm_layer == nn.InstanceNorm2d - - model = [] - - n_downsampling = 2 - mult = 2 ** n_downsampling - for i in range(n_blocks): # add ResNet blocks - model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)] - #torch.Size([1, 256, 32, 32]) - - for i in range(n_downsampling): # add upsampling layers - mult = 2 ** (n_downsampling - i) - # model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), - # kernel_size=3, stride=2, - # padding=1, output_padding=1, - # bias=use_bias), - # norm_layer(int(ngf * mult / 2)), - # nn.ReLU(True)] - #https://distill.pub/2016/deconv-checkerboard/ - #https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/issues/190 - - model += [ nn.Upsample(scale_factor = 2, mode='nearest'), - nn.ReflectionPad2d(1), - nn.Conv2d(ngf * mult, int(ngf * mult / 2),kernel_size=3, stride=1, padding=0), - norm_layer(int(ngf * mult / 2)), - nn.ReLU(True)] - # model += [nn.ReflectionPad2d(3)] - # model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] - # model += [nn.Tanh()] - # model += [nn.Sigmoid()] - - self.model = nn.Sequential(*model) - - def forward(self, input): - """Standard forward""" - return self.model(input) - - - -class conv_3d(nn.Module): - def __init__(self,inchannel,outchannel,kernel_size=3,stride=2,padding=1,norm_layer_3d=nn.BatchNorm3d,use_bias=True): - super(conv_3d, self).__init__() - self.conv = nn.Sequential( - nn.Conv3d(inchannel, outchannel, kernel_size=kernel_size, stride=stride, padding=padding, bias=use_bias), - norm_layer_3d(outchannel), - nn.ReLU(inplace=True), - ) - - def forward(self, x): - x = self.conv(x) - return x - -class conv_2d(nn.Module): - def __init__(self,inchannel,outchannel,kernel_size=3,stride=1,padding=1,norm_layer_2d=nn.BatchNorm2d,use_bias=True): - super(conv_2d, self).__init__() - self.conv = nn.Sequential( - nn.ReflectionPad2d(padding), - nn.Conv2d(inchannel, outchannel, kernel_size=kernel_size, stride=stride, padding=0, bias=use_bias), - norm_layer_2d(outchannel), - nn.ReLU(inplace=True), - ) - - def forward(self, x): - x = self.conv(x) - return x - - -class encoder_3d(nn.Module): - def __init__(self,in_channel,norm_layer_2d,norm_layer_3d,use_bias): - super(encoder_3d, self).__init__() - self.down1 = conv_3d(1, 64, 3, 2, 1,norm_layer_3d,use_bias) - self.down2 = conv_3d(64, 128, 3, 2, 1,norm_layer_3d,use_bias) - self.down3 = conv_3d(128, 256, 3, 1, 1,norm_layer_3d,use_bias) - self.conver2d = nn.Sequential( - nn.Conv2d(256*int(in_channel/4), 256, kernel_size=3, stride=1, padding=1, bias=use_bias), - norm_layer_2d(256), - nn.ReLU(inplace=True), - ) - - - def forward(self, x): - - x = x.view(x.size(0),1,x.size(1),x.size(2),x.size(3)) - x = self.down1(x) - x = self.down2(x) - x = self.down3(x) - - x = x.view(x.size(0),x.size(1)*x.size(2),x.size(3),x.size(4)) - - x = self.conver2d(x) - - return x - - - -class ALL(nn.Module): - def __init__(self, in_channel, out_channel,norm_layer_2d,norm_layer_3d,use_bias): - super(ALL, self).__init__() - - self.encoder_2d = encoder_2d(4,-1,64,norm_layer=norm_layer_2d,n_blocks=9) - self.encoder_3d = encoder_3d(in_channel,norm_layer_2d,norm_layer_3d,use_bias) - self.decoder_2d = decoder_2d(4,3,64,norm_layer=norm_layer_2d,n_blocks=9) - self.shortcut_cov = conv_2d(3,64,7,1,3,norm_layer_2d,use_bias) - self.merge1 = conv_2d(512,256,3,1,1,norm_layer_2d,use_bias) - self.merge2 = nn.Sequential( - conv_2d(128,64,3,1,1,norm_layer_2d,use_bias), - nn.ReflectionPad2d(3), - nn.Conv2d(64, out_channel, kernel_size=7, padding=0), - nn.Tanh() - ) - - def forward(self, x): - - N = int((x.size()[1])/3) - x_2d = torch.cat((x[:,int((N-1)/2)*3:(int((N-1)/2)+1)*3,:,:], x[:,N-1:N,:,:]), 1) - shortcut_2d = x[:,int((N-1)/2)*3:(int((N-1)/2)+1)*3,:,:] - - x_2d = self.encoder_2d(x_2d) - - x_3d = self.encoder_3d(x) - x = torch.cat((x_2d,x_3d),1) - x = self.merge1(x) - x = self.decoder_2d(x) - shortcut_2d = self.shortcut_cov(shortcut_2d) - x = torch.cat((x,shortcut_2d),1) - x = self.merge2(x) - - return x - -def MosaicNet(in_channel, out_channel, norm='batch'): - - if norm == 'batch': - # norm_layer_2d = nn.BatchNorm2d - # norm_layer_3d = nn.BatchNorm3d - norm_layer_2d = functools.partial(nn.BatchNorm2d, affine=True, track_running_stats=True) - norm_layer_3d = functools.partial(nn.BatchNorm3d, affine=True, track_running_stats=True) - use_bias = False - elif norm == 'instance': - norm_layer_2d = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False) - norm_layer_3d = functools.partial(nn.InstanceNorm3d, affine=False, track_running_stats=False) - use_bias = True - - return ALL(in_channel, out_channel, norm_layer_2d, norm_layer_3d, use_bias) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..23c0f2c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +opencv_python==4.5.1.48 +numpy==1.19.2 +torchvision==0.8.2 +torch==1.7.1 +matplotlib==3.3.2 +tensorboardX==2.2 +scikit-image==0.17.2 \ No newline at end of file diff --git a/server.py b/server.py new file mode 100644 index 0000000..f76ca1c --- /dev/null +++ b/server.py @@ -0,0 +1,60 @@ +import os +import sys +import traceback +import cv2 +import numpy as np +try: + from cores import Options,core + from util import util + from util import image_processing as impro + from models import loadmodel +except Exception as e: + print(e) + input('Please press any key to exit.\n') + sys.exit(0) + +# python server.py --gpu_id 0 --model_path ./pretrained_models/mosaic/clean_face_HD.pth +opt = Options() +opt.parser.add_argument('--port',type=int,default=4000, help='') +opt = opt.getparse(True) +netM = loadmodel.bisenet(opt,'mosaic') +netG = loadmodel.pix2pix(opt) + +from flask import Flask, request +import base64 +import shutil + +app = Flask(__name__) + +@app.route("/handle", methods=["POST"]) +def handle(): + result = {} + # to opencv img + try: + imgRec = request.form['img'] + imgByte = base64.b64decode(imgRec) + img_np_arr = np.frombuffer(imgByte, np.uint8) + img = cv2.imdecode(img_np_arr, cv2.IMREAD_COLOR) + except Exception as e: + result['img'] = imgRec + result['info'] = 'readfailed' + return result + + # run model + try: + if max(img.shape)>1080: + img = impro.resize(img,720,interpolation=cv2.INTER_CUBIC) + img = core.cleanmosaic_img_server(opt,img,netG,netM) + except Exception as e: + result['img'] = imgRec + result['info'] = 'procfailed' + return result + + # return + imgbytes = cv2.imencode('.jpg', img)[1] + imgString = base64.b64encode(imgbytes).decode('utf-8') + result['img'] = imgString + result['info'] = 'ok' + return result + +app.run("0.0.0.0", port= opt.port, debug=opt.debug) \ No newline at end of file diff --git a/train/add/train.py b/train/add/train.py index d64063e..a939ee3 100644 --- a/train/add/train.py +++ b/train/add/train.py @@ -54,10 +54,10 @@ util.writelog(os.path.join(dir_checkpoint,'loss.txt'), str(time.asctime(time.localtime(time.time())))+'\n'+util.opt2str(opt)) -def Totensor(img,use_gpu=True): +def Totensor(img,gpu_id=True): size=img.shape[0] img = torch.from_numpy(img).float() - if opt.use_gpu != -1: + if opt.gpu_id != -1: img = img.cuda() return img @@ -68,11 +68,11 @@ def loadimage(imagepaths,maskpaths,opt,test_flag = False): for i in range(len(imagepaths)): img = impro.resize(impro.imread(imagepaths[i]),opt.loadsize) mask = impro.resize(impro.imread(maskpaths[i],mod = 'gray'),opt.loadsize) - img,mask = data.random_transform_image(img, mask, opt.finesize, test_flag) + img,mask = data.random_transform_pair_image(img, mask, opt.finesize, test_flag) images[i] = (img.transpose((2, 0, 1))/255.0) masks[i] = (mask.reshape(1,1,opt.finesize,opt.finesize)/255.0) - images = Totensor(images,opt.use_gpu) - masks = Totensor(masks,opt.use_gpu) + images = data.to_tensor(images,opt.gpu_id) + masks = data.to_tensor(masks,opt.gpu_id) return images,masks @@ -111,7 +111,7 @@ def loadimage(imagepaths,maskpaths,opt,test_flag = False): f = open(os.path.join(dir_checkpoint,'epoch_log.txt'),'r') opt.startepoch = int(f.read()) f.close() -if opt.use_gpu != -1: +if opt.gpu_id != -1: net.cuda() cudnn.benchmark = True @@ -135,7 +135,7 @@ def loadimage(imagepaths,maskpaths,opt,test_flag = False): starttime = datetime.datetime.now() util.writelog(os.path.join(dir_checkpoint,'loss.txt'),'Epoch {}/{}.'.format(epoch + 1, opt.maxepoch),True) net.train() - if opt.use_gpu != -1: + if opt.gpu_id != -1: net.cuda() epoch_loss = 0 for i in range(int(img_num*0.8/opt.batchsize)): diff --git a/train/clean/train.py b/train/clean/train.py index 70efb41..6ea23b7 100644 --- a/train/clean/train.py +++ b/train/clean/train.py @@ -1,310 +1,241 @@ -import os -import sys -sys.path.append("..") -sys.path.append("../..") -from cores import Options -opt = Options() - -import numpy as np -import cv2 -import random -import torch -import torch.nn as nn -import time -from multiprocessing import Process, Queue - -from util import mosaic,util,ffmpeg,filt,data -from util import image_processing as impro -from models import pix2pix_model,pix2pixHD_model,video_model,unet_model,loadmodel,videoHD_model -import matplotlib -matplotlib.use('Agg') -from matplotlib import pyplot as plt -import torch.backends.cudnn as cudnn - -''' ---------------------------Get options-------------------------- -''' -opt.parser.add_argument('--N',type=int,default=25, help='') -opt.parser.add_argument('--lr',type=float,default=0.0002, help='') -opt.parser.add_argument('--beta1',type=float,default=0.5, help='') -opt.parser.add_argument('--gan', action='store_true', help='if specified, use gan') -opt.parser.add_argument('--l2', action='store_true', help='if specified, use L2 loss') -opt.parser.add_argument('--hd', action='store_true', help='if specified, use HD model') -opt.parser.add_argument('--lambda_L1',type=float,default=100, help='') -opt.parser.add_argument('--lambda_gan',type=float,default=1, help='') -opt.parser.add_argument('--finesize',type=int,default=256, help='') -opt.parser.add_argument('--loadsize',type=int,default=286, help='') -opt.parser.add_argument('--batchsize',type=int,default=1, help='') -opt.parser.add_argument('--norm',type=str,default='instance', help='') -opt.parser.add_argument('--num_D', type=int, default=2, help='number of discriminators to use') -opt.parser.add_argument('--n_layers_D', type=int, default=3, help='only used if which_model_netD==n_layers') -opt.parser.add_argument('--lambda_feat', type=float, default=10.0, help='weight for feature matching loss') -opt.parser.add_argument('--image_pool',type=int,default=8, help='number of image load pool') -opt.parser.add_argument('--load_process',type=int,default=4, help='number of process for loading data') - -opt.parser.add_argument('--dataset',type=str,default='./datasets/face/', help='') -opt.parser.add_argument('--maxiter',type=int,default=10000000, help='') -opt.parser.add_argument('--savefreq',type=int,default=10000, help='') -opt.parser.add_argument('--startiter',type=int,default=0, help='') -opt.parser.add_argument('--continue_train', action='store_true', help='') -opt.parser.add_argument('--savename',type=str,default='face', help='') - - -''' ---------------------------Init-------------------------- -''' -opt = opt.getparse() -dir_checkpoint = os.path.join('checkpoints/',opt.savename) -util.makedirs(dir_checkpoint) -util.writelog(os.path.join(dir_checkpoint,'loss.txt'), - str(time.asctime(time.localtime(time.time())))+'\n'+util.opt2str(opt)) -cudnn.benchmark = True - -N = opt.N -loss_sum = [0.,0.,0.,0.,0.,0] -loss_plot = [[],[],[],[]] -item_plot = [] - -# list video dir -videonames = os.listdir(opt.dataset) -videonames.sort() -lengths = [];tmp = [] -print('Check dataset...') -for video in videonames: - if video != 'opt.txt': - video_images = os.listdir(os.path.join(opt.dataset,video,'origin_image')) - lengths.append(len(video_images)) - tmp.append(video) -videonames = tmp -video_num = len(videonames) - -#--------------------------Init network-------------------------- -print('Init network...') -if opt.hd: - netG = videoHD_model.MosaicNet(3*N+1, 3, norm=opt.norm) -else: - netG = video_model.MosaicNet(3*N+1, 3, norm=opt.norm) -netG.cuda() -loadmodel.show_paramsnumber(netG,'netG') - -if opt.gan: - if opt.hd: - netD = pix2pixHD_model.define_D(6, 64, opt.n_layers_D, norm = opt.norm, use_sigmoid=False, num_D=opt.num_D,getIntermFeat=True) - else: - netD = pix2pix_model.define_D(3*2, 64, 'basic', norm = opt.norm) - netD.cuda() - netD.train() - -#--------------------------continue train-------------------------- -if opt.continue_train: - if not os.path.isfile(os.path.join(dir_checkpoint,'last_G.pth')): - opt.continue_train = False - print('can not load last_G, training on init weight.') -if opt.continue_train: - netG.load_state_dict(torch.load(os.path.join(dir_checkpoint,'last_G.pth'))) - if opt.gan: - netD.load_state_dict(torch.load(os.path.join(dir_checkpoint,'last_D.pth'))) - f = open(os.path.join(dir_checkpoint,'iter'),'r') - opt.startiter = int(f.read()) - f.close() - -#--------------------------optimizer & loss-------------------------- -optimizer_G = torch.optim.Adam(netG.parameters(), lr=opt.lr,betas=(opt.beta1, 0.999)) -criterion_L1 = nn.L1Loss() -criterion_L2 = nn.MSELoss() -if opt.gan: - optimizer_D = torch.optim.Adam(netD.parameters(), lr=opt.lr,betas=(opt.beta1, 0.999)) - if opt.hd: - criterionGAN = pix2pixHD_model.GANLoss(tensor=torch.cuda.FloatTensor).cuda() - criterionFeat = pix2pixHD_model.GAN_Feat_loss(opt) - criterionVGG = pix2pixHD_model.VGGLoss([opt.use_gpu]) - else: - criterionGAN = pix2pix_model.GANLoss(gan_mode='lsgan').cuda() - -''' ---------------------------preload data & data pool-------------------------- -''' -print('Preloading data, please wait...') -def preload(pool): - cnt = 0 - input_imgs = torch.rand(opt.batchsize,N*3+1,opt.finesize,opt.finesize) - ground_trues = torch.rand(opt.batchsize,3,opt.finesize,opt.finesize) - while 1: - try: - for i in range(opt.batchsize): - video_index = random.randint(0,video_num-1) - videoname = videonames[video_index] - img_index = random.randint(int(N/2)+1,lengths[video_index]- int(N/2)-1) - input_imgs[i],ground_trues[i] = data.load_train_video(videoname,img_index,opt) - cnt += 1 - pool.put([input_imgs,ground_trues]) - except Exception as e: - print("Error:",videoname,e) -pool = Queue(opt.image_pool) -for i in range(opt.load_process): - p = Process(target=preload,args=(pool,)) - p.daemon = True - p.start() - -''' ---------------------------train-------------------------- -''' -util.copyfile('./train.py', os.path.join(dir_checkpoint,'train.py')) -util.copyfile('../../models/videoHD_model.py', os.path.join(dir_checkpoint,'model.py')) -netG.train() -time_start=time.time() -print("Begin training...") -for iter in range(opt.startiter+1,opt.maxiter): - - inputdata,target = pool.get() - inputdata,target = inputdata.cuda(),target.cuda() - - if opt.gan: - # compute fake images: G(A) - pred = netG(inputdata) - real_A = inputdata[:,int((N-1)/2)*3:(int((N-1)/2)+1)*3,:,:] - - # --------------------update D-------------------- - pix2pix_model.set_requires_grad(netD,True) - optimizer_D.zero_grad() - # Fake - fake_AB = torch.cat((real_A, pred), 1) - pred_fake = netD(fake_AB.detach()) - loss_D_fake = criterionGAN(pred_fake, False) - # Real - real_AB = torch.cat((real_A, target), 1) - pred_real = netD(real_AB) - loss_D_real = criterionGAN(pred_real, True) - # combine loss and calculate gradients - loss_D = (loss_D_fake + loss_D_real) * 0.5 - loss_sum[4] += loss_D_fake.item() - loss_sum[5] += loss_D_real.item() - # udpate D's weights - loss_D.backward() - optimizer_D.step() - - # --------------------update G-------------------- - pix2pix_model.set_requires_grad(netD,False) - optimizer_G.zero_grad() - - # First, G(A) should fake the discriminator - fake_AB = torch.cat((real_A, pred), 1) - pred_fake = netD(fake_AB) - loss_G_GAN = criterionGAN(pred_fake, True)*opt.lambda_gan - - # combine loss and calculate gradients - if opt.l2: - loss_G_L1 = (criterion_L1(pred, target)+criterion_L2(pred, target)) * opt.lambda_L1 - else: - loss_G_L1 = criterion_L1(pred, target) * opt.lambda_L1 - - if opt.hd: - real_AB = torch.cat((real_A, target), 1) - pred_real = netD(real_AB) - loss_G_GAN_Feat = criterionFeat(pred_fake,pred_real) - loss_VGG = criterionVGG(pred, target) * opt.lambda_feat - loss_G = loss_G_GAN + loss_G_L1 + loss_G_GAN_Feat + loss_VGG - else: - loss_G = loss_G_GAN + loss_G_L1 - loss_sum[0] += loss_G_L1.item() - loss_sum[1] += loss_G_GAN.item() - loss_sum[2] += loss_G_GAN_Feat.item() - loss_sum[3] += loss_VGG.item() - - # udpate G's weights - loss_G.backward() - optimizer_G.step() - - else: - pred = netG(inputdata) - if opt.l2: - loss_G_L1 = (criterion_L1(pred, target)+criterion_L2(pred, target)) * opt.lambda_L1 - else: - loss_G_L1 = criterion_L1(pred, target) * opt.lambda_L1 - loss_sum[0] += loss_G_L1.item() - - optimizer_G.zero_grad() - loss_G_L1.backward() - optimizer_G.step() - - # save train result - if (iter+1)%1000 == 0: - try: - data.showresult(inputdata[:,int((N-1)/2)*3:(int((N-1)/2)+1)*3,:,:], - target, pred, os.path.join(dir_checkpoint,'result_train.jpg')) - except Exception as e: - print(e) - - # plot - if (iter+1)%1000 == 0: - time_end = time.time() - #if opt.gan: - savestr ='iter:{0:d} L1_loss:{1:.3f} GAN_loss:{2:.3f} Feat:{3:.3f} VGG:{4:.3f} time:{5:.2f}'.format( - iter+1,loss_sum[0]/1000,loss_sum[1]/1000,loss_sum[2]/1000,loss_sum[3]/1000,(time_end-time_start)/1000) - util.writelog(os.path.join(dir_checkpoint,'loss.txt'), savestr,True) - if (iter+1)/1000 >= 10: - for i in range(4):loss_plot[i].append(loss_sum[i]/1000) - item_plot.append(iter+1) - try: - labels = ['L1_loss','GAN_loss','GAN_Feat_loss','VGG_loss'] - for i in range(4):plt.plot(item_plot,loss_plot[i],label=labels[i]) - plt.xlabel('iter') - plt.legend(loc=1) - plt.savefig(os.path.join(dir_checkpoint,'loss.jpg')) - plt.close() - except Exception as e: - print("error:",e) - - loss_sum = [0.,0.,0.,0.,0.,0.] - time_start=time.time() - - # save network - if (iter+1)%(opt.savefreq//10) == 0: - torch.save(netG.cpu().state_dict(),os.path.join(dir_checkpoint,'last_G.pth')) - if opt.gan: - torch.save(netD.cpu().state_dict(),os.path.join(dir_checkpoint,'last_D.pth')) - if opt.use_gpu !=-1 : - netG.cuda() - if opt.gan: - netD.cuda() - f = open(os.path.join(dir_checkpoint,'iter'),'w+') - f.write(str(iter+1)) - f.close() - - if (iter+1)%opt.savefreq == 0: - os.rename(os.path.join(dir_checkpoint,'last_G.pth'),os.path.join(dir_checkpoint,str(iter+1)+'G.pth')) - if opt.gan: - os.rename(os.path.join(dir_checkpoint,'last_D.pth'),os.path.join(dir_checkpoint,str(iter+1)+'D.pth')) - print('network saved.') - - #test - if (iter+1)%opt.savefreq == 0: - if os.path.isdir('./test'): - netG.eval() - - test_names = os.listdir('./test') - test_names.sort() - result = np.zeros((opt.finesize*2,opt.finesize*len(test_names),3), dtype='uint8') - - for cnt,test_name in enumerate(test_names,0): - img_names = os.listdir(os.path.join('./test',test_name,'image')) - img_names.sort() - inputdata = np.zeros((opt.finesize,opt.finesize,3*N+1), dtype='uint8') - for i in range(0,N): - img = impro.imread(os.path.join('./test',test_name,'image',img_names[i])) - img = impro.resize(img,opt.finesize) - inputdata[:,:,i*3:(i+1)*3] = img - - mask = impro.imread(os.path.join('./test',test_name,'mask.png'),'gray') - mask = impro.resize(mask,opt.finesize) - mask = impro.mask_threshold(mask,15,128) - inputdata[:,:,-1] = mask - result[0:opt.finesize,opt.finesize*cnt:opt.finesize*(cnt+1),:] = inputdata[:,:,int((N-1)/2)*3:(int((N-1)/2)+1)*3] - inputdata = data.im2tensor(inputdata,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1 = False) - pred = netG(inputdata) - - pred = data.tensor2im(pred,rgb2bgr = False, is0_1 = False) - result[opt.finesize:opt.finesize*2,opt.finesize*cnt:opt.finesize*(cnt+1),:] = pred - - cv2.imwrite(os.path.join(dir_checkpoint,str(iter+1)+'_test.jpg'), result) - netG.train() \ No newline at end of file +import os +import sys +sys.path.append("..") +sys.path.append("../..") +from cores import Options +opt = Options() + +import numpy as np +import cv2 +import random +import torch +import torch.nn as nn +import time + +from util import util,data,dataloader +from util import image_processing as impro +from models import BVDNet,model_util +from skimage.metrics import structural_similarity +from tensorboardX import SummaryWriter + +''' +--------------------------Get options-------------------------- +''' +opt.parser.add_argument('--N',type=int,default=2, help='The input tensor shape is H×W×T×C, T = 2N+1') +opt.parser.add_argument('--S',type=int,default=3, help='Stride of 3 frames') +# opt.parser.add_argument('--T',type=int,default=7, help='T = 2N+1') +opt.parser.add_argument('--M',type=int,default=100, help='How many frames read from each videos') +opt.parser.add_argument('--lr',type=float,default=0.0002, help='') +opt.parser.add_argument('--beta1',type=float,default=0.9, help='') +opt.parser.add_argument('--beta2',type=float,default=0.999, help='') +opt.parser.add_argument('--finesize',type=int,default=256, help='') +opt.parser.add_argument('--loadsize',type=int,default=286, help='') +opt.parser.add_argument('--batchsize',type=int,default=1, help='') +opt.parser.add_argument('--no_gan', action='store_true', help='if specified, do not use gan') +opt.parser.add_argument('--n_blocks',type=int,default=4, help='') +opt.parser.add_argument('--n_layers_D',type=int,default=2, help='') +opt.parser.add_argument('--num_D',type=int,default=3, help='') +opt.parser.add_argument('--lambda_L2',type=float,default=100, help='') +opt.parser.add_argument('--lambda_VGG',type=float,default=1, help='') +opt.parser.add_argument('--lambda_GAN',type=float,default=0.01, help='') +opt.parser.add_argument('--lambda_D',type=float,default=1, help='') +opt.parser.add_argument('--load_thread',type=int,default=16, help='number of thread for loading data') + +opt.parser.add_argument('--dataset',type=str,default='./datasets/face/', help='') +opt.parser.add_argument('--dataset_test',type=str,default='./datasets/face_test/', help='') +opt.parser.add_argument('--n_epoch',type=int,default=200, help='') +opt.parser.add_argument('--save_freq',type=int,default=10000, help='') +opt.parser.add_argument('--continue_train', action='store_true', help='') +opt.parser.add_argument('--savename',type=str,default='face', help='') +opt.parser.add_argument('--showresult_freq',type=int,default=1000, help='') +opt.parser.add_argument('--showresult_num',type=int,default=4, help='') + +def ImageQualityEvaluation(tensor1,tensor2,showiter,writer,tag): + batch_len = len(tensor1) + psnr,ssmi = 0,0 + for i in range(len(tensor1)): + img1,img2 = data.tensor2im(tensor1,rgb2bgr=False,batch_index=i), data.tensor2im(tensor2,rgb2bgr=False,batch_index=i) + psnr += impro.psnr(img1,img2) + ssmi += structural_similarity(img1,img2,multichannel=True) + writer.add_scalars('quality/psnr', {tag:psnr/batch_len}, showiter) + writer.add_scalars('quality/ssmi', {tag:ssmi/batch_len}, showiter) + return psnr/batch_len,ssmi/batch_len + +def ShowImage(tensor1,tensor2,tensor3,showiter,max_num,writer,tag): + show_imgs = [] + for i in range(max_num): + show_imgs += [ data.tensor2im(tensor1,rgb2bgr = False,batch_index=i), + data.tensor2im(tensor2,rgb2bgr = False,batch_index=i), + data.tensor2im(tensor3,rgb2bgr = False,batch_index=i)] + show_img = impro.splice(show_imgs, (opt.showresult_num,3)) + writer.add_image(tag, show_img,showiter,dataformats='HWC') + +''' +--------------------------Init-------------------------- +''' +opt = opt.getparse() +opt.T = 2*opt.N+1 +if opt.showresult_num >opt.batchsize: + opt.showresult_num = opt.batchsize +dir_checkpoint = os.path.join('checkpoints',opt.savename) +util.makedirs(dir_checkpoint) +# start tensorboard +localtime = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime()) +tensorboard_savedir = os.path.join('checkpoints/tensorboard',localtime+'_'+opt.savename) +TBGlobalWriter = SummaryWriter(tensorboard_savedir) +print('Please run "tensorboard --logdir checkpoints/tensorboardX --host=your_server_ip" and input "'+localtime+'" to filter outputs') + +''' +--------------------------Init Network-------------------------- +''' +if opt.gpu_id != '-1' and len(opt.gpu_id) == 1: + torch.backends.cudnn.benchmark = True + +netG = BVDNet.define_G(opt.N,opt.n_blocks,gpu_id=opt.gpu_id) +optimizer_G = torch.optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, opt.beta2)) +lossfun_L2 = nn.MSELoss() +lossfun_VGG = model_util.VGGLoss(opt.gpu_id) +if not opt.no_gan: + netD = BVDNet.define_D(n_layers_D=opt.n_layers_D,num_D=opt.num_D,gpu_id=opt.gpu_id) + optimizer_D = torch.optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, opt.beta2)) + lossfun_GAND = BVDNet.GANLoss('D') + lossfun_GANG = BVDNet.GANLoss('G') + +''' +--------------------------Init DataLoader-------------------------- +''' +videolist_tmp = os.listdir(opt.dataset) +videolist = [] +for video in videolist_tmp: + if os.path.isdir(os.path.join(opt.dataset,video)): + if len(os.listdir(os.path.join(opt.dataset,video,'mask')))>=opt.M: + videolist.append(video) +sorted(videolist) +videolist_train = videolist[:int(len(videolist)*0.8)].copy() +videolist_eval = videolist[int(len(videolist)*0.8):].copy() + +Videodataloader_train = dataloader.VideoDataLoader(opt, videolist_train) +Videodataloader_eval = dataloader.VideoDataLoader(opt, videolist_eval) + +''' +--------------------------Train-------------------------- +''' +previous_predframe_tmp = 0 +for train_iter in range(Videodataloader_train.n_iter): + t_start = time.time() + # train + ori_stream,mosaic_stream,previous_frame = Videodataloader_train.get_data() + ori_stream = data.to_tensor(ori_stream, opt.gpu_id) + mosaic_stream = data.to_tensor(mosaic_stream, opt.gpu_id) + if previous_frame is None: + previous_frame = data.to_tensor(previous_predframe_tmp, opt.gpu_id) + else: + previous_frame = data.to_tensor(previous_frame, opt.gpu_id) + + ############### Forward #################### + # Fake Generator + out = netG(mosaic_stream,previous_frame) + # Discriminator + if not opt.no_gan: + dis_real = netD(torch.cat((mosaic_stream[:,:,opt.N],ori_stream[:,:,opt.N].detach()),dim=1)) + dis_fake_D = netD(torch.cat((mosaic_stream[:,:,opt.N],out.detach()),dim=1)) + loss_D = lossfun_GAND(dis_fake_D,dis_real) * opt.lambda_GAN * opt.lambda_D + # Generator + loss_L2 = lossfun_L2(out,ori_stream[:,:,opt.N]) * opt.lambda_L2 + loss_VGG = lossfun_VGG(out,ori_stream[:,:,opt.N]) * opt.lambda_VGG + loss_G = loss_L2+loss_VGG + if not opt.no_gan: + dis_fake_G = netD(torch.cat((mosaic_stream[:,:,opt.N],out),dim=1)) + loss_GANG = lossfun_GANG(dis_fake_G) * opt.lambda_GAN + loss_G = loss_G + loss_GANG + + ############### Backward Pass #################### + optimizer_G.zero_grad() + loss_G.backward() + optimizer_G.step() + + if not opt.no_gan: + optimizer_D.zero_grad() + loss_D.backward() + optimizer_D.step() + + previous_predframe_tmp = out.detach().cpu().numpy() + + if not opt.no_gan: + TBGlobalWriter.add_scalars('loss/train', {'L2':loss_L2.item(),'VGG':loss_VGG.item(), + 'loss_D':loss_D.item(),'loss_G':loss_G.item()}, train_iter) + else: + TBGlobalWriter.add_scalars('loss/train', {'L2':loss_L2.item(),'VGG':loss_VGG.item()}, train_iter) + + # save network + if train_iter%opt.save_freq == 0 and train_iter != 0: + model_util.save(netG, os.path.join('checkpoints',opt.savename,str(train_iter)+'_G.pth'), opt.gpu_id) + if not opt.no_gan: + model_util.save(netD, os.path.join('checkpoints',opt.savename,str(train_iter)+'_D.pth'), opt.gpu_id) + + # Image quality evaluation + if train_iter%(opt.showresult_freq//10) == 0: + ImageQualityEvaluation(out,ori_stream[:,:,opt.N],train_iter,TBGlobalWriter,'train') + + # Show result + if train_iter % opt.showresult_freq == 0: + ShowImage(mosaic_stream[:,:,opt.N],out,ori_stream[:,:,opt.N],train_iter,opt.showresult_num,TBGlobalWriter,'train') + + ''' + --------------------------Eval-------------------------- + ''' + if (train_iter)%5 ==0: + ori_stream,mosaic_stream,previous_frame = Videodataloader_eval.get_data() + ori_stream = data.to_tensor(ori_stream, opt.gpu_id) + mosaic_stream = data.to_tensor(mosaic_stream, opt.gpu_id) + if previous_frame is None: + previous_frame = data.to_tensor(previous_predframe_tmp, opt.gpu_id) + else: + previous_frame = data.to_tensor(previous_frame, opt.gpu_id) + with torch.no_grad(): + out = netG(mosaic_stream,previous_frame) + loss_L2 = lossfun_L2(out,ori_stream[:,:,opt.N]) * opt.lambda_L2 + loss_VGG = lossfun_VGG(out,ori_stream[:,:,opt.N]) * opt.lambda_VGG + #TBGlobalWriter.add_scalars('loss/eval', {'L2':loss_L2.item(),'VGG':loss_VGG.item()}, train_iter) + previous_predframe_tmp = out.detach().cpu().numpy() + + # Image quality evaluation + if train_iter%(opt.showresult_freq//10) == 0: + psnr,ssmi = ImageQualityEvaluation(out,ori_stream[:,:,opt.N],train_iter,TBGlobalWriter,'eval') + + # Show result + if train_iter % opt.showresult_freq == 0: + ShowImage(mosaic_stream[:,:,opt.N],out,ori_stream[:,:,opt.N],train_iter,opt.showresult_num,TBGlobalWriter,'eval') + t_end = time.time() + print('iter:{0:d} t:{1:.2f} L2:{2:.4f} vgg:{3:.4f} psnr:{4:.2f} ssmi:{5:.3f}'.format(train_iter,t_end-t_start, + loss_L2.item(),loss_VGG.item(),psnr,ssmi) ) + t_strat = time.time() + + ''' + --------------------------Test-------------------------- + ''' + if train_iter % opt.showresult_freq == 0 and os.path.isdir(opt.dataset_test): + show_imgs = [] + videos = os.listdir(opt.dataset_test) + sorted(videos) + for video in videos: + frames = os.listdir(os.path.join(opt.dataset_test,video,'image')) + sorted(frames) + for step in range(5): + mosaic_stream = [] + for i in range(opt.T): + _mosaic = impro.imread(os.path.join(opt.dataset_test,video,'image',frames[i*opt.S+step]),loadsize=opt.finesize,rgb=True) + mosaic_stream.append(_mosaic) + if step == 0: + previous = impro.imread(os.path.join(opt.dataset_test,video,'image',frames[opt.N*opt.S-1]),loadsize=opt.finesize,rgb=True) + previous = data.im2tensor(previous,bgr2rgb = False, gpu_id = opt.gpu_id, is0_1 = False) + mosaic_stream = (np.array(mosaic_stream).astype(np.float32)/255.0-0.5)/0.5 + mosaic_stream = mosaic_stream.reshape(1,opt.T,opt.finesize,opt.finesize,3).transpose((0,4,1,2,3)) + mosaic_stream = data.to_tensor(mosaic_stream, opt.gpu_id) + with torch.no_grad(): + out = netG(mosaic_stream,previous) + previous = out + show_imgs+= [data.tensor2im(mosaic_stream[:,:,opt.N],rgb2bgr = False),data.tensor2im(out,rgb2bgr = False)] + + show_img = impro.splice(show_imgs, (len(videos),2)) + TBGlobalWriter.add_image('test', show_img,train_iter,dataformats='HWC') \ No newline at end of file diff --git a/util/data.py b/util/data.py index c76dfa3..8a7865e 100755 --- a/util/data.py +++ b/util/data.py @@ -1,20 +1,31 @@ import random import os +from util.mosaic import get_random_parameter import numpy as np import torch import torchvision.transforms as transforms import cv2 from . import image_processing as impro -from . import mosaic -transform = transforms.Compose([ - transforms.ToTensor(), - transforms.Normalize(mean = (0.5, 0.5, 0.5), std = (0.5, 0.5, 0.5)) - ] -) - -def tensor2im(image_tensor, imtype=np.uint8, gray=False, rgb2bgr = True ,is0_1 = False): +from . import degradater + +def to_tensor(data,gpu_id): + data = torch.from_numpy(data) + if gpu_id != '-1': + data = data.cuda() + return data + +def normalize(data): + ''' + normalize to -1 ~ 1 + ''' + return (data.astype(np.float32)/255.0-0.5)/0.5 + +def anti_normalize(data): + return np.clip((data*0.5+0.5)*255,0,255).astype(np.uint8) + +def tensor2im(image_tensor, gray=False, rgb2bgr = True ,is0_1 = False, batch_index=0): image_tensor =image_tensor.data - image_numpy = image_tensor[0].cpu().float().numpy() + image_numpy = image_tensor[batch_index].cpu().float().numpy() if not is0_1: image_numpy = (image_numpy + 1)/2.0 @@ -24,7 +35,7 @@ def tensor2im(image_tensor, imtype=np.uint8, gray=False, rgb2bgr = True ,is0_1 = if gray: h, w = image_numpy.shape[1:] image_numpy = image_numpy.reshape(h,w) - return image_numpy.astype(imtype) + return image_numpy.astype(np.uint8) # output 3ch if image_numpy.shape[0] == 1: @@ -32,11 +43,10 @@ def tensor2im(image_tensor, imtype=np.uint8, gray=False, rgb2bgr = True ,is0_1 = image_numpy = image_numpy.transpose((1, 2, 0)) if rgb2bgr and not gray: image_numpy = image_numpy[...,::-1]-np.zeros_like(image_numpy) - return image_numpy.astype(imtype) + return image_numpy.astype(np.uint8) -def im2tensor(image_numpy, imtype=np.uint8, gray=False,bgr2rgb = True, reshape = True, use_gpu = 0, use_transform = True,is0_1 = True): - +def im2tensor(image_numpy, gray=False,bgr2rgb = True, reshape = True, gpu_id = '-1',is0_1 = False): if gray: h, w = image_numpy.shape image_numpy = (image_numpy/255.0-0.5)/0.5 @@ -47,18 +57,15 @@ def im2tensor(image_numpy, imtype=np.uint8, gray=False,bgr2rgb = True, reshape = h, w ,ch = image_numpy.shape if bgr2rgb: image_numpy = image_numpy[...,::-1]-np.zeros_like(image_numpy) - if use_transform: - image_tensor = transform(image_numpy) + if is0_1: + image_numpy = image_numpy/255.0 else: - if is0_1: - image_numpy = image_numpy/255.0 - else: - image_numpy = (image_numpy/255.0-0.5)/0.5 - image_numpy = image_numpy.transpose((2, 0, 1)) - image_tensor = torch.from_numpy(image_numpy).float() + image_numpy = (image_numpy/255.0-0.5)/0.5 + image_numpy = image_numpy.transpose((2, 0, 1)) + image_tensor = torch.from_numpy(image_numpy).float() if reshape: image_tensor = image_tensor.reshape(1,ch,h,w) - if use_gpu != -1: + if gpu_id != '-1': image_tensor = image_tensor.cuda() return image_tensor @@ -68,53 +75,7 @@ def shuffledata(data,target): np.random.set_state(state) np.random.shuffle(target) -def random_transform_video(src,target,finesize,N): - #random blur - if random.random()<0.2: - h,w = src.shape[:2] - src = src[:8*(h//8),:8*(w//8)] - Q_ran = random.randint(1,15) - src[:,:,:3*N] = impro.dctblur(src[:,:,:3*N],Q_ran) - target = impro.dctblur(target,Q_ran) - - #random crop - h,w = target.shape[:2] - h_move = int((h-finesize)*random.random()) - w_move = int((w-finesize)*random.random()) - target = target[h_move:h_move+finesize,w_move:w_move+finesize,:] - src = src[h_move:h_move+finesize,w_move:w_move+finesize,:] - - #random flip - if random.random()<0.5: - src = src[:,::-1,:] - target = target[:,::-1,:] - - #random color - alpha = random.uniform(-0.1,0.1) - beta = random.uniform(-0.1,0.1) - b = random.uniform(-0.05,0.05) - g = random.uniform(-0.05,0.05) - r = random.uniform(-0.05,0.05) - for i in range(N): - src[:,:,i*3:(i+1)*3] = impro.color_adjust(src[:,:,i*3:(i+1)*3],alpha,beta,b,g,r) - target = impro.color_adjust(target,alpha,beta,b,g,r) - - #random resize blur - if random.random()<0.5: - interpolations = [cv2.INTER_LINEAR,cv2.INTER_CUBIC,cv2.INTER_LANCZOS4] - size_ran = random.uniform(0.7,1.5) - interpolation_up = interpolations[random.randint(0,2)] - interpolation_down =interpolations[random.randint(0,2)] - - tmp = cv2.resize(src[:,:,:3*N], (int(finesize*size_ran),int(finesize*size_ran)),interpolation=interpolation_up) - src[:,:,:3*N] = cv2.resize(tmp, (finesize,finesize),interpolation=interpolation_down) - - tmp = cv2.resize(target, (int(finesize*size_ran),int(finesize*size_ran)),interpolation=interpolation_up) - target = cv2.resize(tmp, (finesize,finesize),interpolation=interpolation_down) - - return src,target - -def random_transform_single(img,out_shape): +def random_transform_single_mask(img,out_shape): out_h,out_w = out_shape img = cv2.resize(img,(int(out_w*random.uniform(1.1, 1.5)),int(out_h*random.uniform(1.1, 1.5)))) h,w = img.shape[:2] @@ -130,90 +91,65 @@ def random_transform_single(img,out_shape): img = cv2.resize(img,(out_w,out_h)) return img -def random_transform_image(img,mask,finesize,test_flag = False): - #random scale - if random.random()<0.5: - h,w = img.shape[:2] - loadsize = min((h,w)) - a = (float(h)/float(w))*random.uniform(0.9, 1.1) - if h B,C,T,H,W + self.ori_stream = self.ori_stream.reshape (1,self.opt.T,opt.finesize,opt.finesize,3).transpose((0,4,1,2,3)) + self.mosaic_stream = self.mosaic_stream.reshape(1,self.opt.T,opt.finesize,opt.finesize,3).transpose((0,4,1,2,3)) + + #Init frist previous frame + self.previous_pred = self.ori_load_pool[self.opt.S*self.opt.N-1].copy() + # previous B,C,H,W + self.previous_pred = self.previous_pred.reshape(1,opt.finesize,opt.finesize,3).transpose((0,3,1,2)) + + def normalize(self,data): + ''' + normalize to -1 ~ 1 + ''' + return (data.astype(np.float32)/255.0-0.5)/0.5 + + def anti_normalize(self,data): + return np.clip((data*0.5+0.5)*255,0,255).astype(np.uint8) + + def next(self): + # random + if np.random.random()<0.05: + self.startpos = [random.randint(0,self.mosaic_size),random.randint(0,self.mosaic_size)] + if np.random.random()<0.02: + self.transform_params['rate']['crop'] = [np.random.random(),np.random.random()] + if np.random.random()<0.02: + self.loadsize = np.random.randint(self.opt.finesize,self.opt.loadsize) + + if self.t != 0: + self.previous_pred = None + self.ori_load_pool [:self.opt.S*self.opt.T-1] = self.ori_load_pool [1:self.opt.S*self.opt.T] + self.mosaic_load_pool[:self.opt.S*self.opt.T-1] = self.mosaic_load_pool[1:self.opt.S*self.opt.T] + #print(os.path.join(self.video_dir,'origin_image','%05d' % (self.opt.S*self.opt.T+self.t)+'.jpg')) + _ori_img = impro.imread(os.path.join(self.video_dir,'origin_image','%05d' % (self.opt.S*self.opt.T+self.t)+'.jpg'),loadsize=self.loadsize,rgb=True) + _mask = impro.imread(os.path.join(self.video_dir,'mask','%05d' % (self.opt.S*self.opt.T+self.t)+'.png' ),mod='gray',loadsize=self.loadsize) + _mosaic_img = mosaic.addmosaic_base(_ori_img, _mask, self.mosaic_size,0, self.mod,self.rect_rat,self.feather,self.startpos) + _ori_img = data.random_transform_single_image(_ori_img,self.opt.finesize,self.transform_params) + _mosaic_img = data.random_transform_single_image(_mosaic_img,self.opt.finesize,self.transform_params) + + _ori_img,_mosaic_img = self.normalize(_ori_img),self.normalize(_mosaic_img) + self.ori_load_pool [self.opt.S*self.opt.T-1] = _ori_img + self.mosaic_load_pool[self.opt.S*self.opt.T-1] = _mosaic_img + + self.ori_stream = self.ori_load_pool [np.linspace(0, (self.opt.T-1)*self.opt.S,self.opt.T,dtype=np.int64)].copy() + self.mosaic_stream = self.mosaic_load_pool[np.linspace(0, (self.opt.T-1)*self.opt.S,self.opt.T,dtype=np.int64)].copy() + + # stream B,T,H,W,C -> B,C,T,H,W + self.ori_stream = self.ori_stream.reshape (1,self.opt.T,self.opt.finesize,self.opt.finesize,3).transpose((0,4,1,2,3)) + self.mosaic_stream = self.mosaic_stream.reshape(1,self.opt.T,self.opt.finesize,self.opt.finesize,3).transpose((0,4,1,2,3)) + + self.t += 1 + +class VideoDataLoader(object): + """VideoDataLoader""" + def __init__(self, opt, videolist, test_flag=False): + super(VideoDataLoader, self).__init__() + self.videolist = [] + self.opt = opt + self.test_flag = test_flag + for i in range(self.opt.n_epoch): + self.videolist += videolist.copy() + random.shuffle(self.videolist) + self.each_video_n_iter = self.opt.M -self.opt.S*(self.opt.T+1) + self.n_iter = len(self.videolist)//self.opt.load_thread//self.opt.batchsize*self.each_video_n_iter*self.opt.load_thread + self.queue = Queue(self.opt.load_thread) + self.ori_stream = np.zeros((self.opt.batchsize,3,self.opt.T,self.opt.finesize,self.opt.finesize),dtype=np.float32)# B,C,T,H,W + self.mosaic_stream = np.zeros((self.opt.batchsize,3,self.opt.T,self.opt.finesize,self.opt.finesize),dtype=np.float32)# B,C,T,H,W + self.previous_pred = np.zeros((self.opt.batchsize,3,self.opt.finesize,self.opt.finesize),dtype=np.float32) + self.load_init() + + def load(self,videolist): + for load_video_iter in range(len(videolist)//self.opt.batchsize): + iter_videolist = videolist[load_video_iter*self.opt.batchsize:(load_video_iter+1)*self.opt.batchsize] + videoloaders = [VideoLoader(self.opt,os.path.join(self.opt.dataset,iter_videolist[i]),self.test_flag) for i in range(self.opt.batchsize)] + for each_video_iter in range(self.each_video_n_iter): + for i in range(self.opt.batchsize): + self.ori_stream[i] = videoloaders[i].ori_stream + self.mosaic_stream[i] = videoloaders[i].mosaic_stream + if each_video_iter == 0: + self.previous_pred[i] = videoloaders[i].previous_pred + videoloaders[i].next() + if each_video_iter == 0: + self.queue.put([self.ori_stream.copy(),self.mosaic_stream.copy(),self.previous_pred]) + else: + self.queue.put([self.ori_stream.copy(),self.mosaic_stream.copy(),None]) + + def load_init(self): + ptvn = len(self.videolist)//self.opt.load_thread #pre_thread_video_num + for i in range(self.opt.load_thread): + p = Process(target=self.load,args=(self.videolist[i*ptvn:(i+1)*ptvn],)) + p.daemon = True + p.start() + + def get_data(self): + return self.queue.get() \ No newline at end of file diff --git a/util/degradater.py b/util/degradater.py new file mode 100644 index 0000000..9e4d227 --- /dev/null +++ b/util/degradater.py @@ -0,0 +1,119 @@ +''' +https://github.com/sonack/GFRNet_pytorch_new +''' +import random +import cv2 +import numpy as np + +def gaussian_blur(img, sigma=3, size=13): + if sigma > 0: + if isinstance(size, int): + size = (size, size) + img = cv2.GaussianBlur(img, size, sigma) + return img + +def down(img, scale, shape): + if scale > 1: + h, w, _ = shape + scaled_h, scaled_w = int(h / scale), int(w / scale) + img = cv2.resize(img, (scaled_w, scaled_h), interpolation = cv2.INTER_CUBIC) + return img + +def up(img, scale, shape): + if scale > 1: + h, w, _ = shape + img = cv2.resize(img, (w, h), interpolation = cv2.INTER_CUBIC) + return img + +def awgn(img, level): + if level > 0: + noise = np.random.randn(*img.shape) * level + img = (img + noise).clip(0,255).astype(np.uint8) + return img + +def jpeg_compressor(img,quality): + if quality > 0: # 0 indicating no lossy compression (i.e losslessly compression) + encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), quality] + img = cv2.imdecode(cv2.imencode('.jpg', img, encode_param)[1], 1) + return img + +def get_random_degenerate_params(mod='strong'): + ''' + mod : strong | only_downsample | only_4x | weaker_1 | weaker_2 + ''' + params = {} + gaussianBlur_size_list = list(range(3,14,2)) + + if mod == 'strong': + gaussianBlur_sigma_list = [1 + x for x in range(3)] + gaussianBlur_sigma_list += [0] + downsample_scale_list = [1 + x * 0.1 for x in range(0,71)] + awgn_level_list = list(range(1, 8, 1)) + jpeg_quality_list = list(range(10, 41, 1)) + jpeg_quality_list += int(len(jpeg_quality_list) * 0.33) * [0] + + elif mod == 'only_downsample': + gaussianBlur_sigma_list = [0] + downsample_scale_list = [1 + x * 0.1 for x in range(0,71)] + awgn_level_list = [0] + jpeg_quality_list = [0] + + elif mod == 'only_4x': + gaussianBlur_sigma_list = [0] + downsample_scale_list = [4] + awgn_level_list = [0] + jpeg_quality_list = [0] + + elif mod == 'weaker_1': # 0.5 trigger prob + gaussianBlur_sigma_list = [1 + x for x in range(3)] + gaussianBlur_sigma_list += int(len(gaussianBlur_sigma_list)) * [0] # 1/2 trigger this degradation + + downsample_scale_list = [1 + x * 0.1 for x in range(0,71)] + downsample_scale_list += int(len(downsample_scale_list)) * [1] + + awgn_level_list = list(range(1, 8, 1)) + awgn_level_list += int(len(awgn_level_list)) * [0] + + jpeg_quality_list = list(range(10, 41, 1)) + jpeg_quality_list += int(len(jpeg_quality_list)) * [0] + + elif mod == 'weaker_2': # weaker than weaker_1, jpeg [20,40] + gaussianBlur_sigma_list = [1 + x for x in range(3)] + gaussianBlur_sigma_list += int(len(gaussianBlur_sigma_list)) * [0] # 1/2 trigger this degradation + + downsample_scale_list = [1 + x * 0.1 for x in range(0,71)] + downsample_scale_list += int(len(downsample_scale_list)) * [1] + + awgn_level_list = list(range(1, 8, 1)) + awgn_level_list += int(len(awgn_level_list)) * [0] + + jpeg_quality_list = list(range(20, 41, 1)) + jpeg_quality_list += int(len(jpeg_quality_list)) * [0] + + params['blur_sigma'] = random.choice(gaussianBlur_sigma_list) + params['blur_size'] = random.choice(gaussianBlur_size_list) + params['updown_scale'] = random.choice(downsample_scale_list) + params['awgn_level'] = random.choice(awgn_level_list) + params['jpeg_quality'] = random.choice(jpeg_quality_list) + + return params + +def degradate(img,params,jpeg_last = True): + shape = img.shape + if not params: + params = get_random_degenerate_params('original') + + if jpeg_last: + img = gaussian_blur(img,params['blur_sigma'],params['blur_size']) + img = down(img,params['updown_scale'],shape) + img = awgn(img,params['awgn_level']) + img = up(img,params['updown_scale'],shape) + img = jpeg_compressor(img,params['jpeg_quality']) + else: + img = gaussian_blur(img,params['blur_sigma'],params['blur_size']) + img = down(img,params['updown_scale'],shape) + img = awgn(img,params['awgn_level']) + img = jpeg_compressor(img,params['jpeg_quality']) + img = up(img,params['updown_scale'],shape) + + return img \ No newline at end of file diff --git a/util/ffmpeg.py b/util/ffmpeg.py index 4eb668b..2088142 100755 --- a/util/ffmpeg.py +++ b/util/ffmpeg.py @@ -1,5 +1,5 @@ import os,json - +import subprocess # ffmpeg 3.4.6 def args2cmd(args): @@ -32,10 +32,11 @@ def run(args,mode = 0): return sout def video2image(videopath, imagepath, fps=0, start_time='00:00:00', last_time='00:00:00'): - args = ['ffmpeg', '-i', '"'+videopath+'"'] + args = ['ffmpeg'] if last_time != '00:00:00': args += ['-ss', start_time] args += ['-t', last_time] + args += ['-i', '"'+videopath+'"'] if fps != 0: args += ['-r', str(fps)] args += ['-f', 'image2','-q:v','-0',imagepath] diff --git a/util/image_processing.py b/util/image_processing.py index 722a000..6f1dd91 100755 --- a/util/image_processing.py +++ b/util/image_processing.py @@ -8,16 +8,7 @@ if 'Windows' in platform.platform(): system_type = 'Windows' -DCT_Q = np.array([[8,16,19,22,26,27,29,34], - [16,16,22,24,27,29,34,37], - [19,22,26,27,29,34,34,38], - [22,22,26,27,29,34,37,40], - [22,26,27,29,32,35,40,48], - [26,27,29,32,35,40,48,58], - [26,27,29,34,38,46,56,59], - [27,29,35,38,46,56,69,83]]) - -def imread(file_path,mod = 'normal',loadsize = 0): +def imread(file_path,mod = 'normal',loadsize = 0, rgb=False): ''' mod: 'normal' | 'gray' | 'all' loadsize: 0->original @@ -43,6 +34,9 @@ def imread(file_path,mod = 'normal',loadsize = 0): if loadsize != 0: img = resize(img, loadsize, interpolation=cv2.INTER_CUBIC) + if rgb and img.ndim==3: + img = img[:,:,::-1] + return img def imwrite(file_path,img): @@ -110,6 +104,12 @@ def color_adjust(img,alpha=0,beta=0,b=0,g=0,r=0,ran = False): return (np.clip(img,0,255)).astype('uint8') +def CAdaIN(src,dst): + ''' + make src has dst's style + ''' + return np.std(dst)*((src-np.mean(src))/np.std(src))+np.mean(dst) + def makedataset(target_image,orgin_image): target_image = resize(target_image,256) orgin_image = resize(orgin_image,256) @@ -118,34 +118,6 @@ def makedataset(target_image,orgin_image): img[0:256,0:256] = target_image[0:256,int(w/2-256/2):int(w/2+256/2)] img[0:256,256:512] = orgin_image[0:256,int(w/2-256/2):int(w/2+256/2)] return img - -def block_dct_and_idct(g,QQF,QQF_16): - return cv2.idct(np.round(16.0*cv2.dct(g)/QQF)*QQF_16) - -def image_dct_and_idct(I,QF): - h,w = I.shape - QQF = DCT_Q*QF - QQF_16 = QQF/16.0 - for i in range(h//8): - for j in range(w//8): - I[i*8:(i+1)*8,j*8:(j+1)*8] = cv2.idct(np.round(16.0*cv2.dct(I[i*8:(i+1)*8,j*8:(j+1)*8])/QQF)*QQF_16) - #I[i*8:(i+1)*8,j*8:(j+1)*8] = block_dct_and_idct(I[i*8:(i+1)*8,j*8:(j+1)*8],QQF,QQF_16) - return I - -def dctblur(img,Q): - ''' - Q: 1~20, 1->best - ''' - h,w = img.shape[:2] - img = img[:8*(h//8),:8*(w//8)] - img = img.astype(np.float32) - if img.ndim == 2: - img = image_dct_and_idct(img, Q) - if img.ndim == 3: - h,w,ch = img.shape - for i in range(ch): - img[:,:,i] = image_dct_and_idct(img[:,:,i], Q) - return (np.clip(img,0,255)).astype(np.uint8) def find_mostlikely_ROI(mask): contours,hierarchy=cv2.findContours(mask, cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE) @@ -211,6 +183,30 @@ def mask_area(mask): except: area = 0 return area +import time +def replace_mosaic(img_origin,img_fake,mask,x,y,size,no_feather): + img_fake = cv2.resize(img_fake,(size*2,size*2),interpolation=cv2.INTER_CUBIC) + if no_feather: + img_origin[y-size:y+size,x-size:x+size]=img_fake + return img_origin + else: + # #color correction + # RGB_origin = img_origin[y-size:y+size,x-size:x+size].mean(0).mean(0) + # RGB_fake = img_fake.mean(0).mean(0) + # for i in range(3):img_fake[:,:,i] = np.clip(img_fake[:,:,i]+RGB_origin[i]-RGB_fake[i],0,255) + #eclosion + eclosion_num = int(size/10)+2 + + mask_crop = cv2.resize(mask,(img_origin.shape[1],img_origin.shape[0]))[y-size:y+size,x-size:x+size] + mask_crop = ch_one2three(mask_crop) + + mask_crop = (cv2.blur(mask_crop, (eclosion_num, eclosion_num))) + mask_crop = mask_crop/255.0 + + img_crop = img_origin[y-size:y+size,x-size:x+size] + img_origin[y-size:y+size,x-size:x+size] = np.clip((img_crop*(1-mask_crop)+img_fake*mask_crop),0,255).astype('uint8') + + return img_origin def Q_lapulase(resImg): @@ -225,31 +221,25 @@ def Q_lapulase(resImg): score = res.var() return score -def replace_mosaic(img_origin,img_fake,mask,x,y,size,no_feather): - img_fake = cv2.resize(img_fake,(size*2,size*2),interpolation=cv2.INTER_LANCZOS4) - if no_feather: - img_origin[y-size:y+size,x-size:x+size]=img_fake - img_result = img_origin - else: - #color correction - RGB_origin = img_origin[y-size:y+size,x-size:x+size].mean(0).mean(0) - RGB_fake = img_fake.mean(0).mean(0) - for i in range(3):img_fake[:,:,i] = np.clip(img_fake[:,:,i]+RGB_origin[i]-RGB_fake[i],0,255) - #eclosion - eclosion_num = int(size/5) - entad = int(eclosion_num/2+2) - - mask = cv2.resize(mask,(img_origin.shape[1],img_origin.shape[0])) - mask = ch_one2three(mask) - - mask = (cv2.blur(mask, (eclosion_num, eclosion_num))) - mask_tmp = np.zeros_like(mask) - mask_tmp[y-size:y+size,x-size:x+size] = mask[y-size:y+size,x-size:x+size]# Fix edge overflow - mask = mask_tmp/255.0 - - img_tmp = np.zeros(img_origin.shape) - img_tmp[y-size:y+size,x-size:x+size]=img_fake - img_result = img_origin.copy() - img_result = (img_origin*(1-mask)+img_tmp*mask).astype('uint8') +def psnr(img1,img2): + mse = np.mean((img1/255.0-img2/255.0)**2) + if mse < 1e-10: + return 100 + psnr_v = 20*np.log10(1/np.sqrt(mse)) + return psnr_v + +def splice(imgs,splice_shape): + '''Stitching multiple images, all imgs must have the same size + imgs : [img1,img2,img3,img4] + splice_shape: (2,2) + ''' + h,w,ch = imgs[0].shape + output = np.zeros((h*splice_shape[0],w*splice_shape[1],ch),np.uint8) + cnt = 0 + for i in range(splice_shape[0]): + for j in range(splice_shape[1]): + if cnt < len(imgs): + output[h*i:h*(i+1),w*j:w*(j+1)] = imgs[cnt] + cnt += 1 + return output - return img_result \ No newline at end of file diff --git a/util/util.py b/util/util.py index 571974e..4952df8 100755 --- a/util/util.py +++ b/util/util.py @@ -1,4 +1,6 @@ import os +import random +import string import shutil def Traversal(filedir): @@ -10,6 +12,9 @@ def Traversal(filedir): Traversal(dir) return file_list +def randomstr(num): + return ''.join(random.sample(string.ascii_letters + string.digits, num)) + def is_img(path): ext = os.path.splitext(path)[1] ext = ext.lower()