eods-example3-query-eods-download-results-mosaic-s2s-with-rasterio.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import eodslib\n",
    "from pathlib import Path\n",
    "import subprocess\n",
    "from dotenv import load_dotenv\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# USER MUST EDIT THE ENVIRONMENT FILE REFERENCED BELOW, OR CREATE THEIR OWN FILE AND REFERENCE IT\n",
    "load_dotenv('sample.env')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set configuration based on contents of the ENVIRONMENT FILE.\n",
    "conn = {\n",
    "    'domain': os.getenv(\"HOST\"),\n",
    "    'username': os.getenv(\"API_USER\"),\n",
    "    'access_token': os.getenv(\"API_TOKEN\"),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use default path to local \"output\" directory\n",
    "output_dir = eodslib.make_output_dir(Path.cwd() / 'output')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example of EODS query for generating a cloud free mosaic for Summer 2020\n",
    "# since no 'geom' parameter is set, this will query every granule reference with the catalogue\n",
    "\n",
    "eods_params = {\n",
    "    'output_dir':output_dir,\n",
    "    'sat_id':2,\n",
    "    'date_start':'2020-05-01',\n",
    "    'date_end':'2020-06-30',\n",
    "    'find_least_cloud':True,\n",
    "    }\n",
    "\n",
    "list_of_layers, df = eodslib.query_catalog(conn, **eods_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is an example where you want to override the list of layers returned by the EODS query cell above\n",
    "list_of_layers = [\n",
    "    'geonode:S2B_20200424_lat55lon215_T30UWF_ORB037_utm30n_osgb_vmsk_sharp_rad_srefdem_stdsref',\n",
    "    'geonode:S2A_20200419_lat55lon061_T30UXF_ORB037_utm30n_osgb_vmsk_sharp_rad_srefdem_stdsref'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an empty list of results\n",
    "list_of_results = list()\n",
    "\n",
    "# iterative through the returned EODS query list of layers\n",
    "for lyr in list_of_layers:\n",
    "\n",
    "    # for each layer, create a wps config dictionary, inserting the layer name\n",
    "    config_wpsprocess = {'template_xml':'gsdownload_template.xml',\n",
    "        'xml_config':{\n",
    "            'template_layer_name':lyr,\n",
    "            'template_outputformat':'image/tiff',\n",
    "            'template_mimetype':'application/zip'\n",
    "                },\n",
    "        'dl_bool':True\n",
    "    }\n",
    "\n",
    "    # for each layer, call the run wps function which:\n",
    "        # submits the WPS job\n",
    "        # polls the wps server until the job is READY\n",
    "        # downloads the result (and retries up to 3 times if there's a break in the download connection)\n",
    "    execution_dict = eodslib.run_wps(conn, config_wpsprocess, output_dir=output_dir, verify=True)\n",
    "\n",
    "    # append the execution dictionary to the results list\n",
    "    list_of_results.append(execution_dict)\n",
    "\n",
    "# log the output to a log csv file\n",
    "eodslib.output_log(list_of_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grab the s2 output files\n",
    "rio_merge_cmd = [str(input_file) for input_file in output_dir.glob('**/*osgb_vmsk_sharp_rad_srefdem_stdsref.tiff')]\n",
    "\n",
    "# build a list to represent the rio merge command\n",
    "rio_merge_cmd.insert(0,'rio')\n",
    "rio_merge_cmd.insert(1,'merge')\n",
    "rio_merge_cmd.append(str(output_dir / 'merge-tmp.tiff'))\n",
    "\n",
    "# execute the the merge of the downloaded S2s to a single geotiff, note this can be a BIG file ... \n",
    "subprocess.run(rio_merge_cmd, capture_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build a list to represent the rio overview command and execute it with subprocess\n",
    "# this adds overviews to the merged output\n",
    "rio_overviews_cmd = ['rio', 'overview', '--build', '2,4,8,16', str(output_dir / 'merge-tmp.tiff')]\n",
    "subprocess.run(rio_overviews_cmd, capture_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build a list to represent the rio conver command and execute it with subprocess,\n",
    "# this adds inner tiling and compression, effectively optimising the rendering in a gis application\n",
    "rio_convert_cmd = [\n",
    "    'rio',\n",
    "    'convert',\n",
    "    str(output_dir / 'merge-tmp.tiff'),\n",
    "    str(output_dir / 'merge.tiff'),\n",
    "    '--co',\n",
    "    'compress=LZW',\n",
    "    '--co',\n",
    "    'tiled=true',\n",
    "    '--co',\n",
    "    'blockxsize=256',\n",
    "    '--co',\n",
    "    'blockysize=256',\n",
    "]\n",
    "subprocess.run(rio_convert_cmd, capture_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop intermediate merged file\n",
    "Path(output_dir / 'merge-tmp.tiff').unlink()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('### Script Finished')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}