Skip to content

Commit

Permalink
Style and generate footprints
Browse files Browse the repository at this point in the history
  • Loading branch information
Michael Jendryke committed Dec 6, 2017
1 parent 5618ab3 commit 83ea67a
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 42 deletions.
2 changes: 1 addition & 1 deletion downloadmanager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,4 +222,4 @@ def loadxml(self, xmlfile, orderNumber):
if total_files == count[0][0]: # get the only element that the query returns
return 1
else:
return 0
return 0
10 changes: 5 additions & 5 deletions nyx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ def path(p):

def datadir(dir):
if dir == '':
print('ERROR: Provide a directory like "-d /home/mydata" \
(this should point to the directory with all your order folders')
print('ERROR:\tProvide a Data directory like "-d /home/mydata"')
print('\t(this should point to the directory with all your order folders)')
exit()
elif not os.path.exists(dir):
print('ERROR: Data directory {d} does not exist'.format(d=dir))
Expand All @@ -64,8 +64,8 @@ def datadir(dir):

def workingdir(dir):
if dir == '':
print('ERROR: Provide a temporary working directory like "-w /tmp" \
(this should point to the directory outside of datadir')
print('ERROR:\tProvide a temporary working directory like "-w /tmp"')
print('\t(this should point to the directory outside of datadir)')
exit()
elif not os.path.exists(dir):
print('ERROR: Working directory {d} does not exist'.format(d=dir))
Expand Down Expand Up @@ -212,4 +212,4 @@ def main(argv):


if __name__ == "__main__":
main(sys.argv[1:])
main(sys.argv[1:])
111 changes: 83 additions & 28 deletions processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,77 @@
except Exception as e:
print(e)
import sql
# from threading import Thread, current_thread
import concurrent.futures
import time
import utilities


class footprint:
def info():
print('Generating Footprints')
print('Generating Footprints')

def generate(datadir, tempdir):
rows = sql.select("SELECT * FROM footprintmissing", '')
print('INFO: {num} images have no Footprint in table imagedata'.format(num=len(rows)))
for row in rows:
orderNumber = str(row[0])
filename = str(row[1])

workingdir = os.path.join(tempdir, 'temp')
if not os.path.isdir(workingdir):
os.mkdir(os.path.expanduser(workingdir))

file = os.path.join(datadir, orderNumber, filename)
if not os.path.isfile(file):
print('WARNING: File {f} is not there'.format(f=file))
else:
print(datadir, workingdir, orderNumber, file)
out = footprint.extract(datadir, workingdir, orderNumber, file)
if os.path.exists(out):
footprint.loadgeomtopgsql(out)
# https://stackoverflow.com/a/15143994/1623867
# ThreadPoolExecutor for I/O bound operations
# ProcessPoolExecutor for CPU bound
multicore = False
start = time.time()
if multicore is True:
executor = concurrent.futures.ProcessPoolExecutor(1)
futures = [executor.submit(
footprint.processor, row, datadir, tempdir
) for row in rows]
concurrent.futures.wait(futures)
else:
for row in rows:
footprint.processor(row, datadir, tempdir)
end = time.time()
print(end - start)

def try_my_operation(row, datadir, tempdir):
try:
print('ID: {ID} {row} {dd} {td}'.format(
ID=os.getpid(), row=row, dd=datadir, td=tempdir
))
except:
print('error with item')

def processor(row, datadir, tempdir):
print('INFO: Process ID: {ID} {row} {dd} {td}'.format(
ID=os.getpid(),
row=row,
dd=datadir,
td=tempdir
))
orderNumber = str(row[0])
filename = str(row[1])
noaaid = str(row[2])

workingdir = os.path.join(tempdir, 'temp')
if not os.path.isdir(workingdir):
os.mkdir(os.path.expanduser(workingdir))

def extract(basedir, workingdir, orderNumber, file):
file = os.path.join(datadir, orderNumber, filename)
if not os.path.isfile(file):
print('WARNING: File {f} is not there'.format(f=file))
else:
print(workingdir, orderNumber, file, filename, noaaid)

out = footprint.extract(workingdir, orderNumber, file)
# This will load the shape to the database if the file exists
if os.path.exists(out):
footprint.loadgeomtopgsql(out, filename, noaaid, orderNumber)
utilities.filesandfolders.deletefiles(workingdir)

def extract(workingdir, orderNumber, file):
layer = '//All_Data/VIIRS-DNB-SDR_All/Radiance'

# gdal_translate -of GTiff HDF5:"D:\TEMP\noaa2\GDNBO-SVDNB_npp_d20171125_t2100535_e2106339_b31503_c20171128024404585872_nobc_ops.h5"://All_Data/VIIRS-DNB-SDR_All/Radiance test.tif
infile = 'HDF5:"{file}":{layer}'.format(file=file, layer=layer)
r1 = '{u}{end}'.format(u=str(uuid.uuid4()), end='.tif')
outfile = os.path.join(workingdir, r1) # this should be different from basedir
outfile = os.path.join(workingdir, r1) # this should be different from basedir
gdaltranslate = '{tool} {of} {infile} {outfile}'.format(
tool='gdal_translate',
of='-of GTiff',
Expand All @@ -54,7 +92,7 @@ def extract(basedir, workingdir, orderNumber, file):
outfile = os.path.join(workingdir, r2)
gdalwarp = '{tool} {param} {of} {infile} {outfile}'.format(
tool='gdalwarp',
param='-dstnodata 0 -dstalpha',
param='-ot Int16 -wt Int16 -dstnodata 0 -dstalpha',
of='-of GTiff',
infile=infile,
outfile=outfile
Expand All @@ -73,7 +111,7 @@ def extract(basedir, workingdir, orderNumber, file):
param2='-b 2 -f "ESRI Shapefile"',
outfile=outfile
)

print(polygonize)
# shell=True is very unsecure we have to fix this!
subprocess.check_call(polygonize, shell=True)
Expand All @@ -89,14 +127,14 @@ def extract(basedir, workingdir, orderNumber, file):
tool='ogr2ogr',
outfile=outfile,
infile=infile,
param='-simplify 10.0'
param='-simplify 0.2'
)
print(simplify)
subprocess.check_call(simplify)

return outfile

def loadgeomtopgsql(file):
def loadgeomtopgsql(file, filename, noaaid, orderNumber):
reader = ogr.Open(file)
layer = reader.GetLayer(0)
try:
Expand All @@ -106,7 +144,24 @@ def loadgeomtopgsql(file):

for i in range(layer.GetFeatureCount()):
feature = layer.GetFeature(i)
#print(feature.ExportToJson())
print(feature.geometry())
SQL = "INSERT INTO imagedata"
#INSERT INTO imagedata(footprint) SELECT ST_GeomFromText('POLYGON ((47.8986402364685 -19.9761359737374,77.2019166143206 -24.5331415521829,75.348830485111 -44.4051468911004,38.8567335982238 -38.6872585624496,47.8986402364685 -19.9761359737374))',4326)
# print(feature.ExportToJson())
geom = feature.geometry()
SQL = "INSERT INTO {table}(file_name, noaaid, orderNumber, footprint) SELECT '{fn}', {ni}, {on}, ST_GeomFromText('{geom}',{epsg})".format(
table='imagedata',
fn=filename,
ni=noaaid,
on=orderNumber,
geom=str(geom),
epsg=str(epsg)
)
data = ('',)
try:
sql.insert(SQL,data)
except Exception as e:
raise
else:
pass
finally:
pass

# INSERT INTO imagedata(footprint) SELECT ST_GeomFromText('POLYGON ((47.8986402364685 -19.9761359737374,77.2019166143206 -24.5331415521829,75.348830485111 -44.4051468911004,38.8567335982238 -38.6872585624496,47.8986402364685 -19.9761359737374))',4326)
2 changes: 1 addition & 1 deletion sql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,4 +111,4 @@ def ordercomplete(o):
r = bool(cur.fetchall()[0][0])
conn.commit()
disconnect(conn, cur)
return r
return r
14 changes: 7 additions & 7 deletions utilities.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ def freespace(d):
r = 1
return r

def getFolderSize(d):
total_size = 0
for dirpath, dirnames, filenames in os.walk(d):
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
return total_size
def getFolderSize(d):
total_size = 0
for dirpath, dirnames, filenames in os.walk(d):
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
return total_size

def getFileSize(d):
try:
Expand Down

0 comments on commit 83ea67a

Please sign in to comment.