-
Notifications
You must be signed in to change notification settings - Fork 0
/
submit.py
232 lines (192 loc) · 10.1 KB
/
submit.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
import os
import sys
import matplotlib as mpl
import numpy as np
from tqdm import tqdm
from PIL import Image
if os.environ.get('DISPLAY','') == '':
print('WARNING: No display found. Using non-interactive Agg backend for loading matplotlib.')
mpl.use('Agg')
from matplotlib import pyplot as plt
import torch
from optparse import OptionParser
from torchvision.transforms import transforms
import config
from predict import predict
from unet.unet_model import UNetResNet
from tensorboardX import SummaryWriter
def submit(net, writer):
torch.no_grad()
"""Used for Kaggle submission: predicts and encode all test images"""
directory_list = os.listdir(config.DIRECTORY_TEST)
if not os.path.exists(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG):
os.makedirs(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG)
if os.path.exists(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG + ".csv"):
os.remove(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG + ".csv")
print("WARNING: delete file '{}'".format(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG + ".csv"))
with open(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG + ".csv", 'a') as f:
f.write('id,rle_mask\n')
for index, img_name in enumerate(tqdm(directory_list)):
if config.DIRECTORY_SUFFIX_IMG not in img_name: continue
img = Image.open(config.DIRECTORY_TEST + img_name).convert('RGB')
img_n = config.PREDICT_TRANSFORM_IMG(img).unsqueeze(0) # add N
mask_pred = predict(net, img_n).squeeze(0) # reduce N
"""if config.TRAIN_GPU: """
masks_pred_pil = config.PREDICT_TRANSFORM_BACK(mask_pred) # return gray scale PIL
masks_pred_np = np.where(np.asarray(masks_pred_pil, order="F") > config.TRAIN_CHOSEN_THRESHOLD, 1, 0) # return tensor with (H, W) - proved
enc = rle_encode(masks_pred_np)
f.write('{},{}\n'.format(img_name.replace(config.DIRECTORY_SUFFIX_MASK, ""), enc))
if index % 100 == 0:
F = plt.figure()
plt.subplot(221)
plt.imshow(img)
plt.title("Image_Real")
plt.grid(False)
plt.subplot(222)
plt.imshow(masks_pred_pil)
plt.title("Result")
plt.grid(False)
plt.subplot(223)
plt.imshow(config.tensor_to_PIL(mask_pred))
plt.title("Predicted")
plt.grid(False)
plt.subplot(224)
plt.imshow(Image.fromarray(np.where(masks_pred_np > config.TRAIN_CHOSEN_THRESHOLD, 255, 0), mode="L"))
plt.title("Encoded")
plt.grid(False)
writer.add_figure(config.PREDICTION_TAG + "/" + str(img_name), F, global_step=index)
if config.PREDICTION_SAVE_IMG: masks_pred_pil.save(config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG + "/" + img_name)
def get_args():
parser = OptionParser()
parser.add_option('-g', '--gpu', dest='gpu', default="", help='use cuda, please put all gpu id here')
parser.add_option('-c', '--load', dest='load', default=False, help='load file model')
parser.add_option('-p', '--dir_prefix', dest='dir_prefix', default="", help='the root directory')
parser.add_option('-t', '--tag', dest='tag', default="", help='tag for tensorboard-log')
(options, args) = parser.parse_args()
return options
def rle_encoding(img):
if len(img.shape) != 2 or img.shape[0] == 1:
print("WARNING: The Image shape is {}, expected (H, W).".format(img.shape))
pixels = img.flatten().astype(dtype=np.byte)
if (pixels[0]) != 0 and (pixels[0]) != 1:
print("WARNING: The Image Start with non-binary value. Expected 0 or 1, got {}.".format(pixels[0]))
dots = np.where(img.T.flatten() == 1)[0]
run_lengths = []
prev = -2
for b in dots:
if (b>prev+1): run_lengths.extend((b + 1, 0))
run_lengths[-1] += 1
prev = b
return ' '.join(map(str, run_lengths))
def rle_encode(img):
if len(img.shape) != 2 or img.shape[0] == 1:
print("WARNING: The Image shape is {}, expected (H, W).".format(img.shape))
pixels = img.flatten(order = 'F')
if 255 in img:
print("WARNING: The Image Start with non-binary value. Expected 0 or 1, got {}.".format(pixels[0]))
print("Here is an example of the image: {}".format(img))
# We avoid issues with '1' at the start or end (at the corners of
# the original image) by setting those pixels to '0' explicitly.
# We do not expect these to be non-zero for an accurate mask,
# so this should not harm the score.
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] = runs[1::2] - runs[:-1:2]
return ' '.join(str(x) for x in runs)
def rle(img):
'''
img: numpy array, 1 - mask, 0 - background
Returns run length as string formated
'''
bytes = np.where(img.flatten(order = 'F') == 1)[0]
runs = []
prev = -2
for b in bytes:
if (b > prev + 1): runs.extend((b + 1, 0))
runs[-1] += 1
prev = b
return ' '.join([str(i) for i in runs])
# credits to https://stackoverflow.com/users/6076729/manuel-lagunas
# def rle_encode(mask_image):
# pixels = mask_image.flatten()
# # We avoid issues with '1' at the start or end (at the corners of
# # the original image) by setting those pixels to '0' explicitly.
# # We do not expect these to be non-zero for an accurate mask,
# # so this should not harm the score.
# pixels[0] = 0
# pixels[-1] = 0
# runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
# runs[1::2] = runs[1::2] - runs[:-1:2]
# return ' '.join(map(str, runs))
# # ref.: https://www.kaggle.com/stainsby/fast-tested-rle
# def rle_encode(img):
# '''
# img: numpy array, 1 - mask, 0 - background
# Returns run length as string formated
# '''
# pixels = img.flatten(order = 'F')
# pixels[0] = 0 # add to prevent bug
# pixels[-1] = 0 # add to prevent bug
# pixels = np.concatenate([[0], pixels, [0]])
# runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
# runs[1::2] -= runs[::2]
# return ' '.join(str(x) for x in runs)
if __name__ == '__main__':
args = get_args()
config.TRAIN_LOAD = args.load
config.PREDICTION_LOAD_TAG = config.TRAIN_LOAD.replace("/", "-").replace(".pth", "-")
config.PREDICTION_TAG = config.TRAIN_LOAD.replace("tensorboard-", "").replace("-checkpoints-", "-").replace(".pth", "").replace("tensorboard/", "").replace("/checkpoints/", "-").replace(".pth", "")
if args.tag != "": config.PREDICTION_TAG = config.PREDICTION_TAG + "-" + args.tag
writer = SummaryWriter(config.DIRECTORY_TEST + "predicted/tensorboard")
print("Copy this line to command: " + "python .local/lib/python2.7/site-packages/tensorboard/main.py --logdir=ResUnet/" + config.DIRECTORY_TEST + "predicted/tensorboard" + " --port=6006")
print("Current Directory: " + str(os.getcwd()))
print("Download Model here: " + "ResUnet/" + config.DIRECTORY_TEST + "predicted/" + config.PREDICTION_TAG + ".csv")
print("====================================")
print("Loading Neuronetwork...")
net = UNetResNet(encoder_depth=50, num_classes=1, num_filters=32, dropout_2d=0.2,
pretrained=True, is_deconv=True)
if config.TRAIN_GPU != "": net = torch.nn.DataParallel(net, device_ids=[int(i) for i in config.TRAIN_GPU.split(",")])
if config.TRAIN_LOAD:
net.load_state_dict(torch.load(config.TRAIN_LOAD))
print('Model loaded from {}'.format(config.TRAIN_LOAD))
torch.manual_seed(config.TRAIN_SEED)
if config.TRAIN_GPU != "":
os.environ["CUDA_VISIBLE_DEVICES"] = config.TRAIN_GPU
print('Using GPU: [' + config.TRAIN_GPU + ']')
torch.cuda.manual_seed_all(config.TRAIN_SEED)
net.cuda()
else:
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
try:
submit(net=net, writer=writer)
except KeyboardInterrupt as e:
print(e)
try:
sys.exit(0)
except SystemExit:
os._exit(0)
# t1 = time.time()
# pred_dict = {idx: rle_encode( np.round( downsample(preds_test[i]) > threshold_best ) ) for i, idx in enumerate(tqdm_notebook(test_df.index.values))}
# t2 = time.time()
#
# print(f"Usedtime = {t2-t1} s")
"""
python submit.py --load tensorboard/2018-10-13-19-53-02-729361-test/checkpoints/CP36.pth --tag bronze-here
download: ResUnet/data/test/images/predicted/SUBMISSION-2018-10-14-11-53-06-571963-bronze-here.csv
ResUnet/data/test/images/predicted/2018-10-14-05-12-51-616453-bronze-here/78a68dece6.png
python submit.py --load tensorboard/2018-10-13-19-53-02-991722-success-music3/checkpoints/CP50.pth --tag second
python .local/lib/python2.7/site-packages/tensorboard/main.py --logdir=ResUnet/tensorboard/2018-10-14-13-32-48-325330-train-predict --port=6006
download: ResUnet/data/test/images/predicted/SUBMISSION-2018-10-14-11-53-06-571963-bronze-here.csv
ResUnet/data/test/images/predicted/2018-10-14-05-12-51-616453-bronze-here/78a68dece6.png
python submit.py --load tensorboard/2018-10-17-17-00-26-568369-wednesday-aft/checkpoints/CP13.pth --tag 'submit'
python .local/lib/python2.7/site-packages/tensorboard/main.py --logdir=ResUnet/data/test/images/predicted/tensorboard --port=6006
Download: ResUnet/data/test/images/predicted/2018-10-17-17-00-26-568369-wednesday-aft-CP13-submit.csv
python submit.py --load tensorboard/2018-10-17-19-47-01-207026-wednesday-eve/checkpoints/CP7.pth --tag 'submit'
Download: ResUnet/data/test/images/predicted/2018-10-17-19-47-01-207026-wednesday-eve-CP7-submit.csv
python submit.py --load tensorboard/2018-10-17-19-47-01-207026-wednesday-eve/checkpoints/CP7.pth --tag 'submit2'
python submit.py --load tensorboard/2018-10-17-19-47-01-207026-wednesday-eve/checkpoints/CP7.pth --tag 'submit5'
python .local/lib/python2.7/site-packages/tensorboard/main.py --logdir=ResUnet/data/test
/images/predicted/tensorboard --port=6006
ResUnet/data/test/images/predicted/2018-10-17-19-47-01-207026-wednesday-eve-CP7-submita.csv
"""