Skip to content

Commit

Permalink
Merge pull request MichaelJendryke#22 from MichaelJendryke/dev
Browse files Browse the repository at this point in the history 
now possible to add multiple orders and servers
  • Loading branch information
Michael Jendryke authored Dec 7, 2017
2 parents e53938d + 93358d5 commit 201ef5c
Show file tree
Hide file tree
Showing 6 changed files with 140 additions and 70 deletions.
25 changes: 12 additions & 13 deletions downloadmanager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@


class image:
def download(self):
def download():
SQL = "SELECT * FROM downloadimages" # all that are not finished
data = ('', )
rows = sql.select(SQL, data)
Expand Down Expand Up @@ -58,7 +58,7 @@ def download(self):
if (
# check in the database if the checksum was given, if not, it is non-verified download
(not checksum == '') and
(self.checksumcheck(dest, checksum.replace('-', ''))) and
(image.checksumcheck(dest, checksum.replace('-', ''))) and
(utilities.getFileSize(dest) == filesize)
):
print('INFO: Download size and md5 verified')
Expand All @@ -70,7 +70,7 @@ def download(self):
if sql.ordercomplete(orderNumber) is True:
sql.setOrderStatus(orderNumber, 'FINISHED')

def checksumcheck(self, d, c):
def checksumcheck(d, c):
filechecksum = utilities.md5sum(d)
xmlchecksum = c
if filechecksum == xmlchecksum:
Expand All @@ -80,14 +80,14 @@ def checksumcheck(self, d, c):


class order:
def add(self, orderNumber, server, directory):
def add(orderNumber, server, directory):
# Note: no quotes
SQL = "INSERT INTO orders (ordernumber, status, server,directory) VALUES (%s,%s,%s,%s);"
data = (orderNumber, "NEW", server, directory)
r = sql.insert(SQL, data)
return r

def remove(self, o):
def remove(o):
SQL = "SELECT * FROM deleteorder WHERE ordernumber = %s"
data = (o,)
rows = sql.select(SQL, data)
Expand Down Expand Up @@ -118,8 +118,8 @@ def remove(self, o):


class manifest():
def download(self, u, p, o):
manifestname = self.getName(u)
def download(u, p, o):
manifestname = manifest.getName(u)
if manifestname == '':
print('ERROR: There seems to be no Manifest file for order {number}'.format(number=o))
sql.setOrderStatus(o, 'NOMANIFEST')
Expand All @@ -135,7 +135,7 @@ def download(self, u, p, o):
else:
print('ERROR: There is no Manifest for order {o}'.format(o=o))

def getName(self, u): # url, location, ordernumber, path
def getName(u): # url, location, ordernumber, path
result = ftp.dirlist(u)
# lets print the string on screen
# print(result.decode('iso-8859-1'))
Expand All @@ -159,7 +159,7 @@ def getName(self, u): # url, location, ordernumber, path
manifestname = parts[8]
return manifestname

def process(self):
def process():
SQL = ("SELECT * FROM processmanifest")
data = ('',)
rows = sql.select(SQL, data)
Expand All @@ -169,13 +169,13 @@ def process(self):
path = row[1]
manifest = row[2]
if os.path.exists(os.path.join(path, str(orderNumber), manifest)):
if self.loadxml(os.path.join(path, str(orderNumber), manifest), orderNumber) == 1:
if manifest.loadxml(os.path.join(path, str(orderNumber), manifest), orderNumber) == 1:
sql.setOrderStatus(str(orderNumber), 'READY')
else:
sql.setOrderStatus(str(orderNumber), 'ERROR')
exit()

def loadxml(self, xmlfile, orderNumber):
def loadxml(xmlfile, orderNumber):
print('INFO: Loading XML Manifest file', str(xmlfile), 'into table images')
tree = ET.parse(xmlfile)
root = tree.getroot()
Expand All @@ -192,7 +192,6 @@ def loadxml(self, xmlfile, orderNumber):
expiration_date = datetime.strptime(expiration_date, "%Y-%m-%dT%H:%M:%SZ")
except:
print('ERROR: Cannot read all values in Manifest', str(xmlfile))

try:
checksum = lineitem.find('item/checksum').text
except:
Expand Down Expand Up @@ -222,4 +221,4 @@ def loadxml(self, xmlfile, orderNumber):
if total_files == count[0][0]: # get the only element that the query returns
return 1
else:
return 0
return 0
2 changes: 1 addition & 1 deletion ntl.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@ dependencies:
- setuptools
- wheel
- zlib
- prettytable
- tabulate
41 changes: 28 additions & 13 deletions nyx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def orderNumber(o):
r = o
return r
except ValueError:
print('Provide a valid ORDERNUMBER like "-o 12344256"')
print(o, ' is not a valid ORDERNUMBER, provide a valid ORDERNUMBER like "-o 12344256"')
exit()

def server(l):
Expand All @@ -41,7 +41,7 @@ def server(l):
def path(p):
if p == '':
question = 'Should the data be stored at the default path {p}'.format(p=cfg_path)
answer = utilities.queriies.query_yes_no(question, default='yes')
answer = utilities.queries.query_yes_no(question, default='yes')
if answer == 'yes':
return cfg_path
else:
Expand All @@ -53,8 +53,8 @@ def path(p):

def datadir(dir):
if dir == '':
print('ERROR: Provide a directory like "-d /home/mydata" \
(this should point to the directory with all your order folders')
print('ERROR:\tProvide a Data directory like "-d /home/mydata"')
print('\t(this should point to the directory with all your order folders)')
exit()
elif not os.path.exists(dir):
print('ERROR: Data directory {d} does not exist'.format(d=dir))
Expand All @@ -64,8 +64,8 @@ def datadir(dir):

def workingdir(dir):
if dir == '':
print('ERROR: Provide a temporary working directory like "-w /tmp" \
(this should point to the directory outside of datadir')
print('ERROR:\tProvide a temporary working directory like "-w /tmp"')
print('\t(this should point to the directory outside of datadir)')
exit()
elif not os.path.exists(dir):
print('ERROR: Working directory {d} does not exist'.format(d=dir))
Expand Down Expand Up @@ -106,6 +106,7 @@ def create_arg_parser():
)
parser.add_argument(
'-o', '--orderNumber',
nargs='+',
default="",
help='The Order Number from NOAA CLASS'
)
Expand All @@ -116,6 +117,7 @@ def create_arg_parser():
)
parser.add_argument(
'-l', '--server',
nargs='+',
default="",
choices=[
'ncdc',
Expand Down Expand Up @@ -167,13 +169,26 @@ def main(argv):
print('Something went wrong')
elif mode == 'addOrder':
print('Add a new order')
orderNumber = checkInput.orderNumber(parsed_args.orderNumber)
server = checkInput.server(parsed_args.server)
orderserver = zip(parsed_args.orderNumber, parsed_args.server)
directory = checkInput.path(parsed_args.path)
if downloadmanager.order.add(orderNumber, server, directory) is None:
print('Order added')
else:
print('There was an error, the order has not been added')
for i in orderserver:
orderNumber = checkInput.orderNumber(i[0])
server = checkInput.server(i[1])
question = 'Order {o} from Server {s} will be added at {p}'.format(
o=orderNumber,
s=server,
p=directory
)
answer = utilities.queries.query_yes_no(question)
if answer == 'yes':
try:
downloadmanager.order.add(orderNumber, server, directory)
except:
print('There was an error, the order has not been added')
else:
print('Order will not be added.')
exit()

elif mode == 'getManifest':
print('Get the manifest for NEW orders')
SQL = "SELECT * FROM getmanifest"
Expand Down Expand Up @@ -212,4 +227,4 @@ def main(argv):


if __name__ == "__main__":
main(sys.argv[1:])
main(sys.argv[1:])
111 changes: 83 additions & 28 deletions processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,77 @@
except Exception as e:
print(e)
import sql
# from threading import Thread, current_thread
import concurrent.futures
import time
import utilities


class footprint:
def info():
print('Generating Footprints')
print('Generating Footprints')

def generate(datadir, tempdir):
rows = sql.select("SELECT * FROM footprintmissing", '')
print('INFO: {num} images have no Footprint in table imagedata'.format(num=len(rows)))
for row in rows:
orderNumber = str(row[0])
filename = str(row[1])

workingdir = os.path.join(tempdir, 'temp')
if not os.path.isdir(workingdir):
os.mkdir(os.path.expanduser(workingdir))

file = os.path.join(datadir, orderNumber, filename)
if not os.path.isfile(file):
print('WARNING: File {f} is not there'.format(f=file))
else:
print(datadir, workingdir, orderNumber, file)
out = footprint.extract(datadir, workingdir, orderNumber, file)
if os.path.exists(out):
footprint.loadgeomtopgsql(out)
# https://stackoverflow.com/a/15143994/1623867
# ThreadPoolExecutor for I/O bound operations
# ProcessPoolExecutor for CPU bound
multicore = False
start = time.time()
if multicore is True:
executor = concurrent.futures.ProcessPoolExecutor(1)
futures = [executor.submit(
footprint.processor, row, datadir, tempdir
) for row in rows]
concurrent.futures.wait(futures)
else:
for row in rows:
footprint.processor(row, datadir, tempdir)
end = time.time()
print(end - start)

def try_my_operation(row, datadir, tempdir):
try:
print('ID: {ID} {row} {dd} {td}'.format(
ID=os.getpid(), row=row, dd=datadir, td=tempdir
))
except:
print('error with item')

def processor(row, datadir, tempdir):
print('INFO: Process ID: {ID} {row} {dd} {td}'.format(
ID=os.getpid(),
row=row,
dd=datadir,
td=tempdir
))
orderNumber = str(row[0])
filename = str(row[1])
noaaid = str(row[2])

workingdir = os.path.join(tempdir, 'temp')
if not os.path.isdir(workingdir):
os.mkdir(os.path.expanduser(workingdir))

def extract(basedir, workingdir, orderNumber, file):
file = os.path.join(datadir, orderNumber, filename)
if not os.path.isfile(file):
print('WARNING: File {f} is not there'.format(f=file))
else:
print(workingdir, orderNumber, file, filename, noaaid)

out = footprint.extract(workingdir, orderNumber, file)
# This will load the shape to the database if the file exists
if os.path.exists(out):
footprint.loadgeomtopgsql(out, filename, noaaid, orderNumber)
utilities.filesandfolders.deletefiles(workingdir)

def extract(workingdir, orderNumber, file):
layer = '//All_Data/VIIRS-DNB-SDR_All/Radiance'

# gdal_translate -of GTiff HDF5:"D:\TEMP\noaa2\GDNBO-SVDNB_npp_d20171125_t2100535_e2106339_b31503_c20171128024404585872_nobc_ops.h5"://All_Data/VIIRS-DNB-SDR_All/Radiance test.tif
infile = 'HDF5:"{file}":{layer}'.format(file=file, layer=layer)
r1 = '{u}{end}'.format(u=str(uuid.uuid4()), end='.tif')
outfile = os.path.join(workingdir, r1) # this should be different from basedir
outfile = os.path.join(workingdir, r1) # this should be different from basedir
gdaltranslate = '{tool} {of} {infile} {outfile}'.format(
tool='gdal_translate',
of='-of GTiff',
Expand All @@ -54,7 +92,7 @@ def extract(basedir, workingdir, orderNumber, file):
outfile = os.path.join(workingdir, r2)
gdalwarp = '{tool} {param} {of} {infile} {outfile}'.format(
tool='gdalwarp',
param='-dstnodata 0 -dstalpha',
param='-ot Int16 -wt Int16 -dstnodata 0 -dstalpha',
of='-of GTiff',
infile=infile,
outfile=outfile
Expand All @@ -73,7 +111,7 @@ def extract(basedir, workingdir, orderNumber, file):
param2='-b 2 -f "ESRI Shapefile"',
outfile=outfile
)

print(polygonize)
# shell=True is very unsecure we have to fix this!
subprocess.check_call(polygonize, shell=True)
Expand All @@ -89,14 +127,14 @@ def extract(basedir, workingdir, orderNumber, file):
tool='ogr2ogr',
outfile=outfile,
infile=infile,
param='-simplify 10.0'
param='-simplify 0.2'
)
print(simplify)
subprocess.check_call(simplify)

return outfile

def loadgeomtopgsql(file):
def loadgeomtopgsql(file, filename, noaaid, orderNumber):
reader = ogr.Open(file)
layer = reader.GetLayer(0)
try:
Expand All @@ -106,7 +144,24 @@ def loadgeomtopgsql(file):

for i in range(layer.GetFeatureCount()):
feature = layer.GetFeature(i)
#print(feature.ExportToJson())
print(feature.geometry())
SQL = "INSERT INTO imagedata"
#INSERT INTO imagedata(footprint) SELECT ST_GeomFromText('POLYGON ((47.8986402364685 -19.9761359737374,77.2019166143206 -24.5331415521829,75.348830485111 -44.4051468911004,38.8567335982238 -38.6872585624496,47.8986402364685 -19.9761359737374))',4326)
# print(feature.ExportToJson())
geom = feature.geometry()
SQL = "INSERT INTO {table}(file_name, noaaid, orderNumber, footprint) SELECT '{fn}', {ni}, {on}, ST_GeomFromText('{geom}',{epsg})".format(
table='imagedata',
fn=filename,
ni=noaaid,
on=orderNumber,
geom=str(geom),
epsg=str(epsg)
)
data = ('',)
try:
sql.insert(SQL,data)
except Exception as e:
raise
else:
pass
finally:
pass

# INSERT INTO imagedata(footprint) SELECT ST_GeomFromText('POLYGON ((47.8986402364685 -19.9761359737374,77.2019166143206 -24.5331415521829,75.348830485111 -44.4051468911004,38.8567335982238 -38.6872585624496,47.8986402364685 -19.9761359737374))',4326)
Loading

0 comments on commit 201ef5c

Please sign in to comment.