-
Notifications
You must be signed in to change notification settings - Fork 10
/
mk_expt.py
189 lines (150 loc) · 7.23 KB
/
mk_expt.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import numpy as np
import os
import string
import argparse
from IPython import embed
def error(str):
print(str)
def fileread(str,breakcode='[[BR]]'):
fid = open(str,'r')
a = fid.readlines()
fid.close()
return breakcode.join(a)
def getOpts(expt_name):
opt = getDefaultOpts();
if(expt_name=='example_expt'):
opt['which_algs_paths'] = ['my_alg','baseline_alg']
opt['Nimgs'] = 1000
opt['ut_id'] = 'unset' # set this using http://uniqueturker.myleott.com/
opt['base_url'] = 'https://www.mywebsite.com/example_expt_data/'
opt['instructions_file'] = './instructions_basic.html'
opt['short_instructions_file'] = './short_instructions_basic.html'
opt['consent_file'] = './consent_basic.html'
opt['use_vigilance'] = False
opt['paired'] = True
else:
error('no opts defined for experiment %s'%expt_name)
opt['expt_name'] = expt_name;
return opt
def getDefaultOpts():
opt = {};
opt['expt_name'] = 'unset'
opt['which_algs_paths'] = 'unset' # paths to images generated by algoritms, e.g. {'my_alg','baseline_alg'}
opt['vigilance_path'] = 'vigilance' # path to vigilance images
opt['gt_path'] = 'gt' # path to gt images
opt['Nimgs'] = 100 # number of images to test
opt['Npairs'] = 60 # number of paired comparisons per HIT
opt['Npractice'] = 10 # number of practice trials per HIT (number of non-practice trials is opt['Npairs']-opt['Npractice'])
opt['Nhits'] = 50 # number of HITs per algorithm
opt['vigilance_freq'] = 0.0 # percent of trials that are vigilance tests
opt['use_vigilance'] = False # include vigilance trials (obviously fake images to check that Turkers are paying attention)
opt['ut_id'] = 'unset' # set this using http://uniqueturker.myleott.com/
opt['base_url'] = 'unset' # url where images to test are accessible as "opt['base_url']/n.png", for integers n
opt['instructions_file'] = 'instructions_basic.html' # instructions appear at the beginning of the HIT
opt['short_instructions_file'] = 'short_instructions_basic.html' # short instructions are shown at the top of every trial
opt['consent_file'] = 'consent_basic.html' # informed consent text appears the beginning of the HIT
opt['im_height'] = 384 # dimensions at which to display the stimuli
opt['im_width'] = 384 #
opt['paired'] = True # if True, then fake/n.jpg will be pitted against real/n.jpg; if false, fake/n.jpg will be pitted against real/m.jpg, for random n and m
opt['filename'] = lambda x : '%i.jpg'%x
return opt
def checkOpts(opt):
if(opt['which_algs_paths']=='unset'):
error('must provide a list of algorithms to test');
if(opt['ut_id']=='unset'):
error('must set a unique id for this HIT using http://uniqueturker.myleott.com/');
if(opt['base_url']=='unset'):
error('must provide a url where test images are accessible');
if(opt['instructions_file']=='unset'):
error('must provide a file containing html formatted instructions to display once at start of experiment');
if(opt['short_instructions_file']=='unset'):
error('must provide a file containing html formatted instructions to display on each trial');
if(opt['consent_file']=='unset'):
error('must provide a file containing html formatted infromed consent test, display at start of experiment');
def mk_expt(expt_name):
# expt parameters
opt = getOpts(expt_name)
# check parameters
checkOpts(opt)
# make dir for expt, overwriting if it already exists
if(os.path.exists(opt['expt_name'])):
os.system('rm -r ./%s'%opt['expt_name'])
os.mkdir(opt['expt_name'])
# rng('shuffle')
csv_fname = os.path.join(opt['expt_name'],'expt_input_data.csv');
# make header
head_gt_side = []
head_images_left = []
head_images_right = []
for i in range(opt['Npairs']):
head_gt_side.append('gt_side%d'%i)
head_images_left.append('images_left%d'%i)
head_images_right.append('images_right%d'%i)
head_gt_side = [head_gt_side,]
head_images_left = [head_images_left,]
head_images_right = [head_images_right,]
A = len(opt['which_algs_paths'])
H = opt['Nhits'] # number of hits
I = opt['Nimgs']
N = opt['Npairs'] # number of images per hit
which_alg = np.random.randint(A, size=H*N)
which_ind0 = np.random.randint(I, size=H*N)
which_ind1 = which_ind0 if(opt['paired']) else np.random.randint(I, size=H*N)
which_side = np.random.randint(2, size=H*N)
vigilance = (np.random.rand(H*N) < opt['vigilance_freq']) * opt['use_vigilance']
gt_side = []
images_left = []
images_right = []
for (nn,data) in enumerate(zip(which_alg,which_ind0,which_ind1,which_side,vigilance)):
cur_which_alg, cur_which_ind0, cur_which_ind1, cur_which_side, cur_vigilance = data
cur_alg_name = opt['which_algs_paths'][cur_which_alg] if not cur_vigilance else opt['vigilance_path']
if(cur_which_side==0):
gt_side.append('left')
images_left.append(('%s/'+opt['filename'](cur_which_ind0))%opt['gt_path'])
images_right.append(('%s/'+opt['filename'](cur_which_ind1))%cur_alg_name)
else:
gt_side.append('right')
images_left.append(('%s/'+opt['filename'](cur_which_ind0))%cur_alg_name)
images_right.append(('%s/'+opt['filename'](cur_which_ind1))%opt['gt_path'])
gt_side = np.array(gt_side).reshape((H,N))
images_left = np.array(images_left).reshape((H,N))
images_right = np.array(images_right).reshape((H,N))
head_gt_side = np.array(head_gt_side)
head_images_left = np.array(head_images_left)
head_images_right = np.array(head_images_right)
csv_out = np.concatenate((np.concatenate((head_gt_side, head_images_left, head_images_right),axis=1),
np.concatenate((gt_side, images_left, images_right),axis=1)),axis=0)
fid = open(csv_fname,'w')
for i in range(csv_out.shape[0]):
for j in range(csv_out.shape[1]-1):
fid.write(csv_out[i,j]+',')
fid.write(csv_out[i,-1])
fid.write('\n')
fid.close()
breakcode='[[BR]]'
# html code generator
# embed()
html = fileread('index_template.html',breakcode=breakcode)
html = html.replace('{{UT_ID}}', opt['ut_id'])
html = html.replace('{{BASE_URL}}', opt['base_url'])
html = html.replace('{{INSTRUCTIONS}}', fileread(opt['instructions_file'],breakcode=breakcode))
html = html.replace('{{SHORT_INSTRUCTIONS}}', fileread(opt['short_instructions_file'],breakcode=breakcode))
html = html.replace('{{CONSENT}}', fileread(opt['consent_file'],breakcode=breakcode))
html = html.replace('{{IM_DIV_HEIGHT}}', '%i'%(opt['im_height']+2))
html = html.replace('{{IM_DIV_WIDTH}}', '%i'%(opt['im_width']+2))
html = html.replace('{{IM_HEIGHT}}', '%i'%(opt['im_height']))
html = html.replace('{{IM_WIDTH}}', '%i'%(opt['im_width']))
html = html.replace('{{N_PRACTICE}}', '%i'%(opt['Npractice']))
html = html.replace('{{TOTAL_NUM_IMS}}', '%i'%(opt['Npairs']))
s = (' ').join(['sequence_helper("${gt_side%d}","${images_left%d}","${images_right%d}");\n'%(i,i,i) for i in range(opt['Npairs'])])
html = html.replace('{{SEQUENCE}}', s)
s = []
s = (' ').join(['<input type="hidden" name="selection%d" id="selection%d" value="unset">\n'%(i,i) for i in range(opt['Npairs'])])
html = html.replace('{{SELECTION}}', s)
fid = open(os.path.join(opt['expt_name'],'index.html'),'w')
fid.writelines(html.split(breakcode))
fid.close()
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-n','--name', type=str, default='experiments name')
args = parser.parse_args()
mk_expt(args.name)