{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Dask" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[](https://colab.research.google.com/github/davemlz/spyndex/blob/main/docs/tutorials/dask.ipynb)\n", "" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "It's parallel processing time: Level 6 - `spyndex + dask`!\n", "\n", "Remember to install `spyndex`!" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!pip install -U spyndex" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now, let's start!\n", "\n", "First, import `spyndex` and `dask`:" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import spyndex\n", "import dask" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## `dask.Array`\n", "\n", "In Level 4 we worked with `xarray` and `DataArrays`. Well, let's do the same tutorial but now working in parallel!\n", "\n", "Let's use the `xarray.DataArray` that is stored in the `spyndex` datasets: `sentinel`:" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "da = spyndex.datasets.open(\"sentinel\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As you already know, this data array is very simple. We have 3 dimensions: `band`, `x` and `y`. Each band is one of the 10 m spectral bands of a Sentinel-2 image." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
<xarray.DataArray (band: 4, x: 300, y: 300)>\n",
"array([[[ 299, 276, 280, ..., 510, 516, 521],\n",
" [ 287, 285, 284, ..., 503, 476, 469],\n",
" [ 287, 292, 288, ..., 454, 411, 337],\n",
" ...,\n",
" [ 502, 508, 520, ..., 683, 670, 791],\n",
" [ 486, 518, 532, ..., 688, 696, 693],\n",
" [ 486, 506, 515, ..., 659, 671, 664]],\n",
"\n",
" [[ 469, 446, 466, ..., 695, 711, 728],\n",
" [ 469, 437, 469, ..., 683, 694, 666],\n",
" [ 460, 460, 460, ..., 628, 595, 527],\n",
" ...,\n",
" [ 804, 808, 832, ..., 920, 872, 1023],\n",
" [ 787, 803, 822, ..., 890, 882, 871],\n",
" [ 787, 799, 822, ..., 893, 832, 834]],\n",
"\n",
" [[ 319, 293, 328, ..., 1054, 1090, 1110],\n",
" [ 327, 318, 345, ..., 1044, 1004, 952],\n",
" [ 339, 355, 323, ..., 922, 784, 652],\n",
" ...,\n",
" [1528, 1516, 1516, ..., 1250, 1246, 1420],\n",
" [1470, 1502, 1498, ..., 1316, 1200, 1162],\n",
" [1394, 1480, 1472, ..., 1288, 1144, 1122]],\n",
"\n",
" [[2164, 2128, 2206, ..., 1796, 1837, 1816],\n",
" [2110, 2017, 2228, ..., 1795, 1839, 1788],\n",
" [2050, 2112, 2062, ..., 1816, 1789, 1864],\n",
" ...,\n",
" [1910, 1942, 1942, ..., 2105, 1898, 2102],\n",
" [1836, 1874, 1916, ..., 2075, 1792, 1747],\n",
" [1778, 1844, 1870, ..., 2087, 1830, 1675]]])\n",
"Coordinates:\n",
" * band (band) <U3 'B02' 'B03' 'B04' 'B08'\n",
"Dimensions without coordinates: x, yarray([[[ 299, 276, 280, ..., 510, 516, 521],\n",
" [ 287, 285, 284, ..., 503, 476, 469],\n",
" [ 287, 292, 288, ..., 454, 411, 337],\n",
" ...,\n",
" [ 502, 508, 520, ..., 683, 670, 791],\n",
" [ 486, 518, 532, ..., 688, 696, 693],\n",
" [ 486, 506, 515, ..., 659, 671, 664]],\n",
"\n",
" [[ 469, 446, 466, ..., 695, 711, 728],\n",
" [ 469, 437, 469, ..., 683, 694, 666],\n",
" [ 460, 460, 460, ..., 628, 595, 527],\n",
" ...,\n",
" [ 804, 808, 832, ..., 920, 872, 1023],\n",
" [ 787, 803, 822, ..., 890, 882, 871],\n",
" [ 787, 799, 822, ..., 893, 832, 834]],\n",
"\n",
" [[ 319, 293, 328, ..., 1054, 1090, 1110],\n",
" [ 327, 318, 345, ..., 1044, 1004, 952],\n",
" [ 339, 355, 323, ..., 922, 784, 652],\n",
" ...,\n",
" [1528, 1516, 1516, ..., 1250, 1246, 1420],\n",
" [1470, 1502, 1498, ..., 1316, 1200, 1162],\n",
" [1394, 1480, 1472, ..., 1288, 1144, 1122]],\n",
"\n",
" [[2164, 2128, 2206, ..., 1796, 1837, 1816],\n",
" [2110, 2017, 2228, ..., 1795, 1839, 1788],\n",
" [2050, 2112, 2062, ..., 1816, 1789, 1864],\n",
" ...,\n",
" [1910, 1942, 1942, ..., 2105, 1898, 2102],\n",
" [1836, 1874, 1916, ..., 2075, 1792, 1747],\n",
" [1778, 1844, 1870, ..., 2087, 1830, 1675]]])array(['B02', 'B03', 'B04', 'B08'], dtype='<U3')
<xarray.DataArray (band: 4, x: 300, y: 300)>\n",
"dask.array<xarray-<this-array>, shape=(4, 300, 300), dtype=float64, chunksize=(1, 100, 100), chunktype=numpy.ndarray>\n",
"Coordinates:\n",
" * band (band) <U3 'B02' 'B03' 'B04' 'B08'\n",
"Dimensions without coordinates: x, y\n",
"
| \n",
" \n", " \n", " | \n", "
array(['B02', 'B03', 'B04', 'B08'], dtype='<U3')
<xarray.DataArray (x: 300, y: 300)>\n",
"dask.array<truediv, shape=(300, 300), dtype=float64, chunksize=(100, 100), chunktype=numpy.ndarray>\n",
"Dimensions without coordinates: x, y\n",
"
| \n",
" \n", " \n", " | \n", "
<xarray.DataArray (x: 300, y: 300)>\n",
"array([[0.74305276, 0.75795126, 0.74112076, ..., 0.26035088, 0.25521011,\n",
" 0.24128503],\n",
" [0.73163726, 0.72762313, 0.73183055, ..., 0.26452976, 0.29370383,\n",
" 0.30510949],\n",
" [0.71619925, 0.71220105, 0.72914046, ..., 0.3265157 , 0.39059464,\n",
" 0.48171701],\n",
" ...,\n",
" [0.11111111, 0.1231926 , 0.1231926 , ..., 0.25484352, 0.20737913,\n",
" 0.19363998],\n",
" [0.1107078 , 0.11018957, 0.12243702, ..., 0.22382778, 0.19786096,\n",
" 0.20110003],\n",
" [0.12105927, 0.10950662, 0.11909037, ..., 0.23674074, 0.23066577,\n",
" 0.19771183]])\n",
"Dimensions without coordinates: x, yarray([[0.74305276, 0.75795126, 0.74112076, ..., 0.26035088, 0.25521011,\n",
" 0.24128503],\n",
" [0.73163726, 0.72762313, 0.73183055, ..., 0.26452976, 0.29370383,\n",
" 0.30510949],\n",
" [0.71619925, 0.71220105, 0.72914046, ..., 0.3265157 , 0.39059464,\n",
" 0.48171701],\n",
" ...,\n",
" [0.11111111, 0.1231926 , 0.1231926 , ..., 0.25484352, 0.20737913,\n",
" 0.19363998],\n",
" [0.1107078 , 0.11018957, 0.12243702, ..., 0.22382778, 0.19786096,\n",
" 0.20110003],\n",
" [0.12105927, 0.10950662, 0.11909037, ..., 0.23674074, 0.23066577,\n",
" 0.19771183]])| \n", " | SR_B1 | \n", "SR_B2 | \n", "SR_B3 | \n", "SR_B4 | \n", "SR_B5 | \n", "SR_B6 | \n", "SR_B7 | \n", "ST_B10 | \n", "class | \n", "
|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "0.089850 | \n", "0.100795 | \n", "0.132227 | \n", "0.165764 | \n", "0.269054 | \n", "0.306206 | \n", "0.251949 | \n", "297.328396 | \n", "Urban | \n", "
| 1 | \n", "0.073859 | \n", "0.086990 | \n", "0.124404 | \n", "0.160979 | \n", "0.281264 | \n", "0.267596 | \n", "0.217917 | \n", "297.107934 | \n", "Urban | \n", "
| 2 | \n", "0.072938 | \n", "0.086028 | \n", "0.120994 | \n", "0.140203 | \n", "0.284220 | \n", "0.258384 | \n", "0.200098 | \n", "297.436064 | \n", "Urban | \n", "
| 3 | \n", "0.087733 | \n", "0.103916 | \n", "0.135981 | \n", "0.163976 | \n", "0.254479 | \n", "0.259580 | \n", "0.216735 | \n", "297.203638 | \n", "Urban | \n", "
| 4 | \n", "0.090593 | \n", "0.109306 | \n", "0.150350 | \n", "0.181260 | \n", "0.269535 | \n", "0.273234 | \n", "0.219554 | \n", "297.097680 | \n", "Urban | \n", "
| \n", " | SR_B1 | \n", "SR_B2 | \n", "SR_B3 | \n", "SR_B4 | \n", "SR_B5 | \n", "SR_B6 | \n", "SR_B7 | \n", "ST_B10 | \n", "class | \n", "
|---|---|---|---|---|---|---|---|---|---|
| npartitions=10 | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 0 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "object | \n", "
| 109 | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 89 | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 99 | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| \n", " | SR_B1 | \n", "SR_B2 | \n", "SR_B3 | \n", "SR_B4 | \n", "SR_B5 | \n", "SR_B6 | \n", "SR_B7 | \n", "ST_B10 | \n", "class | \n", "NDVI | \n", "NDWI | \n", "NDBI | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| npartitions=10 | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 0 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "float64 | \n", "object | \n", "float64 | \n", "float64 | \n", "float64 | \n", "
| 109 | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 89 | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 99 | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| \n", " | SR_B1 | \n", "SR_B2 | \n", "SR_B3 | \n", "SR_B4 | \n", "SR_B5 | \n", "SR_B6 | \n", "SR_B7 | \n", "ST_B10 | \n", "class | \n", "NDVI | \n", "NDWI | \n", "NDBI | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "0.089850 | \n", "0.100795 | \n", "0.132227 | \n", "0.165764 | \n", "0.269054 | \n", "0.306206 | \n", "0.251949 | \n", "297.328396 | \n", "Urban | \n", "0.237548 | \n", "-0.340973 | \n", "0.064584 | \n", "
| 1 | \n", "0.073859 | \n", "0.086990 | \n", "0.124404 | \n", "0.160979 | \n", "0.281264 | \n", "0.267596 | \n", "0.217917 | \n", "297.107934 | \n", "Urban | \n", "0.271989 | \n", "-0.386671 | \n", "-0.024902 | \n", "
| 10 | \n", "0.109801 | \n", "0.124184 | \n", "0.166369 | \n", "0.202229 | \n", "0.317330 | \n", "0.314731 | \n", "0.237525 | \n", "298.668260 | \n", "Urban | \n", "0.221537 | \n", "-0.312098 | \n", "-0.004112 | \n", "
| 100 | \n", "0.022805 | \n", "0.026105 | \n", "0.051735 | \n", "0.034822 | \n", "0.255455 | \n", "0.114627 | \n", "0.054650 | \n", "291.818548 | \n", "Vegetation | \n", "0.760074 | \n", "-0.663173 | \n", "-0.380530 | \n", "
| 101 | \n", "0.021856 | \n", "0.024345 | \n", "0.050635 | \n", "0.034341 | \n", "0.270058 | \n", "0.118175 | \n", "0.052354 | \n", "291.423766 | \n", "Vegetation | \n", "0.774367 | \n", "-0.684215 | \n", "-0.391215 | \n", "