0_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing of raster images using Python (version 3.5): Example using flight of _Acacia dealbata_ \n",
    "\n",
    "## Processing steps include:\n",
    "### - 1: mask images using GDAL utils\n",
    "### - 2: resample a 'slave' raster using a 'master' raster, using GDAL python library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import os, glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare directories\n",
    "inputRasterDir = \"H:/Teja/UAV_mapping_invasive\"\n",
    "outRawDir = \"F:/invasive_spp/raw/accacia_f1/\"\n",
    "os.chdir(inputRasterDir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# list files\n",
    "hyper = inputRasterDir+\"/hyper/acacia_f1/Hyper_A1_biobio1.tif\"\n",
    "rgb = inputRasterDir+\"/rgb/acacia_f1_rgb/flight_01_biobio_ortho003_19s.tif\"\n",
    "structure = \"/structure/biobio_f1_xyz.tif\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_0.25.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_0.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_0.75.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_1.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_1.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_2.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_2.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_3.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_3.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_4.tif']\n",
      "\n",
      "list length : 10\n"
     ]
    }
   ],
   "source": [
    "# textures\n",
    "os.chdir(inputRasterDir+\"/texture\")\n",
    "texture = glob.glob(\"flight_01_biobio*.tif\")\n",
    "texture = [\"H:/Teja/UAV_mapping_invasive/texture/\" + s for s in texture]\n",
    "print(texture)\n",
    "print(\"\")\n",
    "print(\"list length :\", len(texture))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['H:/Teja/UAV_mapping_invasive/hyper/acacia_f1/Hyper_A1_biobio1.tif', 'H:/Teja/UAV_mapping_invasive/rgb/acacia_f1_rgb/flight_01_biobio_ortho003.tif', '/structure/biobio_f1_xyz.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_0.25.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_0.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_0.75.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_1.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_1.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_2.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_2.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_3.5.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_3.tif', 'H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_4.tif']\n",
      "\n",
      "list length : 13\n"
     ]
    }
   ],
   "source": [
    "# append all data\n",
    "list_all = [hyper, rgb, structure] + texture\n",
    "print(list_all)\n",
    "print(\"\")\n",
    "print(\"list length :\", len(list_all))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mask rasters using GDAL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import subprocess to run terminal\n",
    "from subprocess import call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# shapefile used to crop the rasters\n",
    "shp = inputRasterDir+\"/1_reference/acacia_f1_AOI.shp\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mask for : Hyper_A1_biobio1.tif\n",
      "mask for : flight_01_biobio_ortho003.tif\n",
      "mask for : biobio_f1_xyz.tif\n",
      "mask for : flight_01_biobio_ortho003_0.25.tif\n",
      "mask for : flight_01_biobio_ortho003_0.5.tif\n",
      "mask for : flight_01_biobio_ortho003_0.75.tif\n",
      "mask for : flight_01_biobio_ortho003_1.5.tif\n",
      "mask for : flight_01_biobio_ortho003_1.tif\n",
      "mask for : flight_01_biobio_ortho003_2.5.tif\n",
      "mask for : flight_01_biobio_ortho003_2.tif\n",
      "mask for : flight_01_biobio_ortho003_3.5.tif\n",
      "mask for : flight_01_biobio_ortho003_3.tif\n",
      "mask for : flight_01_biobio_ortho003_4.tif\n"
     ]
    }
   ],
   "source": [
    "# run gdalwarp for all rasters\n",
    "for raster in list_all:\n",
    "    baseName = os.path.basename(raster)\n",
    "    process = \"gdalwarp -dstnodata -9999 -cutline \"+shp+\" \"+raster+\" \"+outRawDir+baseName[:-4]+ \"_clip.tif\"\n",
    "    call(process)\n",
    "    print(\"mask for :\", baseName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gdalwarp -dstnodata -9999 -cutline H:/Teja/UAV_mapping_invasive/1_reference/acacia_f1_AOI.shp H:/Teja/UAV_mapping_invasive/texture/flight_01_biobio_ortho003_4.tif F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_4_clip.tif\n"
     ]
    }
   ],
   "source": [
    "print(process)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resample rasters to match the hyperspectral resolution and extention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to resample the \"slave\" raster using a \"master\" raster\n",
    "def reproject_image_to_master( master, slave, res=None ):\n",
    "    import gdal\n",
    "    \n",
    "    slave_ds = gdal.Open( slave )\n",
    "    if slave_ds is None:\n",
    "        raise IOError\n",
    "    slave_proj = slave_ds.GetProjection()\n",
    "    slave_geotrans = slave_ds.GetGeoTransform()\n",
    "    data_type = slave_ds.GetRasterBand(1).DataType\n",
    "    n_bands = slave_ds.RasterCount\n",
    "\n",
    "    master_ds = gdal.Open( master )\n",
    "    if master_ds is None:\n",
    "        raise IOError\n",
    "    master_proj = master_ds.GetProjection()\n",
    "    master_geotrans = master_ds.GetGeoTransform()\n",
    "    w = master_ds.RasterXSize\n",
    "    h = master_ds.RasterYSize\n",
    "    if res is not None:\n",
    "        master_geotrans[1] = float( res )\n",
    "        master_geotrans[-1] = - float ( res )\n",
    "\n",
    "    dst_filename = slave.replace( \".tif\", \"_crop.tif\" )\n",
    "    dst_ds = gdal.GetDriverByName('GTiff').Create(dst_filename, w, h, n_bands, data_type)\n",
    "    dst_ds.SetGeoTransform( master_geotrans )\n",
    "    dst_ds.SetProjection( master_proj)\n",
    "\n",
    "    gdal.ReprojectImage( slave_ds, dst_ds, slave_proj,\n",
    "                         master_proj, gdal.GRA_NearestNeighbour)\n",
    "    dst_ds = None  # Flush to disk\n",
    "    return dst_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_0.25_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_0.5_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_0.75_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_1.5_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_1_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_2.5_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_2_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_3.5_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_3_clip.tif', 'F:/invasive_spp/raw/accacia_f1/flight_01_biobio_ortho003_4_clip.tif', 'F:/invasive_spp/raw/accacia_f1/Hyper_A1_biobio1_clip.tif']\n",
      "\n",
      "length of the list: 11\n"
     ]
    }
   ],
   "source": [
    "# output direction\n",
    "os.chdir(outRawDir)\n",
    "\n",
    "# list of all tiffs\n",
    "list_all = glob.glob(\"*.tif\")\n",
    "list_all = [outRawDir + s for s in list_all]\n",
    "# add folder dir\n",
    "print(list_all)\n",
    "print(\"\")\n",
    "print(\"length of the list:\", len(list_all))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mask for : flight_01_biobio_ortho003_0.5_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_0.75_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_1.5_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_1_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_2.5_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_2_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_3.5_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_3_clip.tif\n",
      "mask for : flight_01_biobio_ortho003_4_clip.tif\n",
      "mask for : Hyper_A1_biobio1_clip.tif\n"
     ]
    }
   ],
   "source": [
    "# resample\n",
    "for raster in list_all[1:]:\n",
    "    baseName = os.path.basename(raster)\n",
    "    reproject_image_to_master(hyper, raster) \n",
    "    # delete the file in the raw folder\n",
    "    os.remove(raster)\n",
    "    print(\"mask for :\", baseName)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}