{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bad674ea",
   "metadata": {},
   "source": [
    "# Extract ERA and ERA5-Land at the Transects"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09e5767a",
   "metadata": {},
   "source": [
    "By Andi Muttaqin (muttaqin@aos.wisc.edu), Nov 2022"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea9fbb13",
   "metadata": {},
   "source": [
    "<b>Input</b>:<br> \n",
    "1. csv_transects_qc02.csv \n",
    "2. csv_transects_qc02.csv <br>\n",
    "\n",
    "<b>Output</b>: \n",
    "1. NetCDF files of extracted reanalysis dataset (.nc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9eb9d014",
   "metadata": {},
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8d48ae9b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T01:52:53.008091Z",
     "start_time": "2022-11-11T01:52:52.861997Z"
    }
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import pandas as pd\n",
    "import metview as mv\n",
    "import xarray as xr\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import binsreg\n",
    "import matplotlib.pyplot as plt\n",
    "from ipynb.fs.full.asm import seg2trans\n",
    "from ipynb.fs.full.asm import load_reg_nc\n",
    "from ipynb.fs.full.asm import binscatter\n",
    "sns.set_theme(style='darkgrid')\n",
    "from cdo import Cdo\n",
    "cdo = Cdo()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a32fbd7a",
   "metadata": {},
   "source": [
    "## Load transects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ae2c98e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T01:52:56.779823Z",
     "start_time": "2022-11-11T01:52:56.667649Z"
    }
   },
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('/bog/amuttaqin/Chapter2_SBC/csv_transects_qc02.csv')\n",
    "df2 = pd.read_csv('/bog/amuttaqin/Chapter2_SBC/csv_transects_qc04.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b9283857",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T01:52:58.900578Z",
     "start_time": "2022-11-11T01:52:58.860082Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>segID</th>\n",
       "      <th>trsID</th>\n",
       "      <th>lat1</th>\n",
       "      <th>lon1</th>\n",
       "      <th>lat2</th>\n",
       "      <th>lon2</th>\n",
       "      <th>lsm1</th>\n",
       "      <th>lsm2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13.7582</td>\n",
       "      <td>98.0249</td>\n",
       "      <td>13.9018</td>\n",
       "      <td>98.3982</td>\n",
       "      <td>0.050076</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14.0382</td>\n",
       "      <td>97.9172</td>\n",
       "      <td>14.1818</td>\n",
       "      <td>98.2905</td>\n",
       "      <td>0.005708</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>14.3182</td>\n",
       "      <td>97.8095</td>\n",
       "      <td>14.4618</td>\n",
       "      <td>98.1828</td>\n",
       "      <td>0.014419</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>14.5982</td>\n",
       "      <td>97.7018</td>\n",
       "      <td>14.7418</td>\n",
       "      <td>98.0751</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9.1179</td>\n",
       "      <td>98.1545</td>\n",
       "      <td>9.0505</td>\n",
       "      <td>98.5487</td>\n",
       "      <td>0.133140</td>\n",
       "      <td>0.98772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>146</td>\n",
       "      <td>515</td>\n",
       "      <td>-2.6573</td>\n",
       "      <td>128.5006</td>\n",
       "      <td>-3.0573</td>\n",
       "      <td>128.5006</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>147</td>\n",
       "      <td>516</td>\n",
       "      <td>-3.7343</td>\n",
       "      <td>130.1365</td>\n",
       "      <td>-3.3717</td>\n",
       "      <td>130.3054</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>516</th>\n",
       "      <td>148</td>\n",
       "      <td>517</td>\n",
       "      <td>0.4917</td>\n",
       "      <td>127.3577</td>\n",
       "      <td>0.5367</td>\n",
       "      <td>127.7552</td>\n",
       "      <td>0.089703</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>149</td>\n",
       "      <td>518</td>\n",
       "      <td>1.3272</td>\n",
       "      <td>128.8892</td>\n",
       "      <td>1.3218</td>\n",
       "      <td>128.4892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>150</td>\n",
       "      <td>519</td>\n",
       "      <td>1.5195</td>\n",
       "      <td>127.3107</td>\n",
       "      <td>1.4173</td>\n",
       "      <td>127.6974</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>519 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     segID  trsID     lat1      lon1     lat2      lon2      lsm1     lsm2\n",
       "0        1      1  13.7582   98.0249  13.9018   98.3982  0.050076  1.00000\n",
       "1        1      2  14.0382   97.9172  14.1818   98.2905  0.005708  1.00000\n",
       "2        1      3  14.3182   97.8095  14.4618   98.1828  0.014419  1.00000\n",
       "3        1      4  14.5982   97.7018  14.7418   98.0751  0.000000  1.00000\n",
       "4        2      5   9.1179   98.1545   9.0505   98.5487  0.133140  0.98772\n",
       "..     ...    ...      ...       ...      ...       ...       ...      ...\n",
       "514    146    515  -2.6573  128.5006  -3.0573  128.5006  0.000000  1.00000\n",
       "515    147    516  -3.7343  130.1365  -3.3717  130.3054  0.000000  1.00000\n",
       "516    148    517   0.4917  127.3577   0.5367  127.7552  0.089703  1.00000\n",
       "517    149    518   1.3272  128.8892   1.3218  128.4892  0.000000  1.00000\n",
       "518    150    519   1.5195  127.3107   1.4173  127.6974  0.000000  1.00000\n",
       "\n",
       "[519 rows x 8 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55d9359d",
   "metadata": {},
   "source": [
    "## Extract Skin Temperature of Ocean and Land, and calculate land-sea temperature difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "17f8dcc8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:32.108347Z",
     "start_time": "2022-11-10T05:39:32.106051Z"
    }
   },
   "outputs": [],
   "source": [
    "# skto = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5/skto/1991-2020_skto.nc\")\n",
    "# sktl = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/skt/1991-2020_skt.nc\")\n",
    "\n",
    "# skto = skto.chunk(\"auto\")\n",
    "# sktl = sktl.chunk(\"auto\")\n",
    "\n",
    "# skto.var235\n",
    "# sktl.var235\n",
    "\n",
    "# lat1 = xr.DataArray(df2[\"lat1\"].to_numpy().tolist(), dims='segment')\n",
    "# lon1 = xr.DataArray(df2[\"lon1\"].to_numpy().tolist(), dims='segment')\n",
    "# lat2 = xr.DataArray(df2[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df2[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# ts_skto = skto.sel(lat=lat1, lon=lon1, method=\"nearest\")\n",
    "# ts_sktl = sktl.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "\n",
    "# ts_skto = ts_skto.compute()\n",
    "# ts_sktl = ts_sktl.compute()\n",
    "\n",
    "# tdiff = ts_sktl - ts_skto\n",
    "# tdiff = tdiff.compute()\n",
    "\n",
    "# tdiff = tdiff.assign_coords(segment=df2['segID'].to_numpy().tolist())\n",
    "# tdiff\n",
    "\n",
    "# tdiff.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_land_sea_skt_diff.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "768105dc",
   "metadata": {},
   "source": [
    "## Extract surface wind at 1000 hPa and synoptic wind at 700 hPa (ERA5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e62f9b72",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:33.214542Z",
     "start_time": "2022-11-10T05:39:33.211762Z"
    }
   },
   "outputs": [],
   "source": [
    "# uv1000 = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5/uv/uv_1000/1991-2020_uv_1000.nc\")\n",
    "# uv700 = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5/uv/uv_700/1991-2020_uv_700.nc\")\n",
    "\n",
    "# uv1000 = uv1000.chunk(\"auto\")\n",
    "# uv700 = uv700.chunk(\"auto\")\n",
    "\n",
    "# lat1 = xr.DataArray(df1[\"lat1\"].to_numpy().tolist(), dims='segment')\n",
    "# lon1 = xr.DataArray(df1[\"lon1\"].to_numpy().tolist(), dims='segment')\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# uv1000_land = uv1000.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "# uv1000_sea  = uv1000.sel(lat=lat1, lon=lon1, method=\"nearest\")\n",
    "# uv700_land  = uv700.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "# uv700_sea   = uv700.sel(lat=lat1, lon=lon1, method=\"nearest\")\n",
    "\n",
    "# uv1000_land.var131\n",
    "\n",
    "# uv1000_land = uv1000_land.compute()\n",
    "# uv1000_sea  = uv1000_sea.compute()\n",
    "# uv700_land  = uv700_land.compute()\n",
    "# uv700_sea   = uv700_sea.compute()\n",
    "\n",
    "# uv1000_land = uv1000_land.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# uv1000_sea  = uv1000_sea.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# uv700_land  = uv700_land.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# uv700_sea   = uv700_sea.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "\n",
    "# uv1000_land = uv1000_land.rename({'var131':'u', 'var132':'v'})\n",
    "# uv1000_sea  = uv1000_sea.rename({'var131':'u', 'var132':'v'})\n",
    "# uv700_land  = uv700_land.rename({'var131':'u', 'var132':'v'})\n",
    "# uv700_sea   = uv700_sea.rename({'var131':'u', 'var132':'v'})\n",
    "\n",
    "# uv1000_land = uv1000_land.drop_vars({'lon', 'lat', 'plev'})\n",
    "# uv1000_sea  = uv1000_sea.drop_vars({'lon', 'lat', 'plev'})\n",
    "# uv700_land  = uv700_land.drop_vars({'lon', 'lat', 'plev'})\n",
    "# uv700_sea   = uv700_sea.drop_vars({'lon', 'lat', 'plev'})\n",
    "\n",
    "# uv1000_land.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_surface_wind_1000_land.nc', 'w')\n",
    "# uv1000_sea.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_surface_wind_1000_sea.nc', 'w')\n",
    "\n",
    "# uv700_land.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_synoptic_wind_700_land.nc', 'w')\n",
    "# uv700_sea.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_synoptic_wind_700_sea.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4715c201",
   "metadata": {},
   "source": [
    "## Extract surface fluxes (ERA5-Land)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9452908",
   "metadata": {},
   "source": [
    "### Volumetric Soil Water Level 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a6dffee7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:35.545688Z",
     "start_time": "2022-11-10T05:39:35.543631Z"
    }
   },
   "outputs": [],
   "source": [
    "# swvl1 = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/swvl1/1991-2020_swvl1.nc\")\n",
    "# swvl1 = swvl1.chunk(\"auto\")\n",
    "\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# swvl1 = swvl1.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "\n",
    "# swvl1 = swvl1.compute()\n",
    "\n",
    "# swvl1 = swvl1.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# swvl1 = swvl1.drop_vars({'lon', 'lat', 'depth_bnds'})\n",
    "# swvl1 = swvl1.squeeze(dim='depth', drop=True)\n",
    "# swvl1\n",
    "\n",
    "# swvl1.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_soil_water_content_1_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd7b2dce",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-02T09:21:21.937125Z",
     "start_time": "2022-11-02T09:21:21.934745Z"
    }
   },
   "source": [
    "### Volumetric Soil Water Level 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b8ae7e83",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:37.804104Z",
     "start_time": "2022-11-10T05:39:37.801810Z"
    }
   },
   "outputs": [],
   "source": [
    "# swvl2 = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/swvl2/1991-2020_swvl2.nc\")\n",
    "# swvl2 = swvl2.chunk(\"auto\")\n",
    "\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# swvl2 = swvl2.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "\n",
    "# swvl2 = swvl2.compute()\n",
    "\n",
    "# swvl2 = swvl2.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# swvl2 = swvl2.drop_vars({'lon', 'lat', 'depth_bnds'})\n",
    "# swvl2 = swvl2.squeeze(dim='depth', drop=True)\n",
    "\n",
    "# swvl2.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_soil_water_content_2_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6d0dd06",
   "metadata": {},
   "source": [
    "### Volumetric Soil Water Level 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "92a69ca7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:39.772248Z",
     "start_time": "2022-11-10T05:39:39.770149Z"
    }
   },
   "outputs": [],
   "source": [
    "# swvl3 = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/swvl3/1991-2020_swvl3.nc\")\n",
    "# swvl3 = swvl3.chunk(\"auto\")\n",
    "\n",
    "# swvl3\n",
    "\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# swvl3 = swvl3.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "\n",
    "# swvl3 = swvl3.compute()\n",
    "\n",
    "# swvl3 = swvl3.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# swvl3 = swvl3.drop_vars({'lon', 'lat', 'depth_bnds'})\n",
    "# swvl3 = swvl3.squeeze(dim='depth', drop=True)\n",
    "\n",
    "# swvl3.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_soil_water_content_3_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d881456",
   "metadata": {},
   "source": [
    "### Volumetric Soil Water Level 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69f8cdec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:41.651148Z",
     "start_time": "2022-11-10T05:39:41.648989Z"
    }
   },
   "outputs": [],
   "source": [
    "# swvl4 = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/swvl4/1991-2020_swvl4.nc\")\n",
    "# swvl4 = swvl4.chunk(\"auto\")\n",
    "\n",
    "# swvl4\n",
    "\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# swvl4 = swvl4.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "\n",
    "# swvl4 = swvl4.compute()\n",
    "\n",
    "# swvl4 = swvl4.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# swvl4 = swvl4.drop_vars({'lon', 'lat', 'depth_bnds'})\n",
    "# swvl4 = swvl4.squeeze(dim='depth', drop=True)\n",
    "\n",
    "# swvl4.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_soil_water_content_4_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd7ccc3c",
   "metadata": {},
   "source": [
    "### Surface latent heat flux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "07be3232",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:43.102141Z",
     "start_time": "2022-11-10T05:39:43.100149Z"
    }
   },
   "outputs": [],
   "source": [
    "# slhf = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/slhf/1991-2020_slhf.nc\")\n",
    "# slhf = slhf.chunk(\"auto\")\n",
    "\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# slhf = slhf.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "# slhf = slhf.compute()\n",
    "\n",
    "# slhf = slhf.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# slhf = slhf.drop_vars({'lon', 'lat'})\n",
    "\n",
    "# slhf.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_surface_latent_heat_flux_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d54c99d",
   "metadata": {},
   "source": [
    "### Surface sensible heat flux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cb4b0bab",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T05:39:43.896047Z",
     "start_time": "2022-11-10T05:39:43.894009Z"
    }
   },
   "outputs": [],
   "source": [
    "# sshf = xr.open_dataset(\"/bog/amuttaqin/Datasets/ERA5-Land/sshf/1991-2020_sshf.nc\")\n",
    "# sshf = sshf.chunk(\"auto\")\n",
    "\n",
    "# lat2 = xr.DataArray(df1[\"lat2\"].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1[\"lon2\"].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# sshf = sshf.sel(lat=lat2, lon=lon2, method=\"nearest\")\n",
    "# sshf = sshf.compute()\n",
    "\n",
    "# sshf = sshf.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# sshf = sshf.drop_vars({'lon', 'lat'})\n",
    "\n",
    "# sshf.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_surface_sensible_heat_flux_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b5b39b5",
   "metadata": {},
   "source": [
    "### Surface net solar radiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "893ab9e7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T10:40:08.543169Z",
     "start_time": "2022-11-10T10:08:09.831266Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/bog/amuttaqin/Datasets/ERA5-Land/ssr/1991-2020_ssr.nc'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # Convert grib file to netcdf\n",
    "\n",
    "# filegrb = \"/bog/amuttaqin/Datasets/ERA5-Land/ssr/1991-2020_ssr.grib\"\n",
    "# filenc = \"/bog/amuttaqin/Datasets/ERA5-Land/ssr/1991-2020_ssr.nc\"\n",
    "\n",
    "# cdo.copy(input=filegrb, options='-f nc4c -z zip', output=filenc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8aa3f935",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T11:10:33.658340Z",
     "start_time": "2022-11-10T10:40:08.573629Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Extract variable at transects\n",
    "# snsr = xr.open_dataset('/bog/amuttaqin/Datasets/ERA5-Land/ssr/1991-2020_ssr.nc')\n",
    "# snsr = snsr.chunk('auto')\n",
    "\n",
    "# lat2 = xr.DataArray(df1['lat2'].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1['lon2'].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# snsr = snsr.sel(lat=lat2, lon=lon2, method='nearest')\n",
    "# snsr = snsr.compute()\n",
    "\n",
    "# snsr = snsr.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# snsr = snsr.drop_vars({'lon', 'lat'})\n",
    "\n",
    "# snsr.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_surface_net_solar_radiation_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bf4ff34",
   "metadata": {},
   "source": [
    "### Boundary layer height"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d6686098",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T11:14:50.940836Z",
     "start_time": "2022-11-10T11:10:33.707826Z"
    }
   },
   "outputs": [],
   "source": [
    "# pblh = xr.open_dataset('/bog/amuttaqin/Datasets/ERA5/pbl_height/1991-2020_blh.nc')\n",
    "# pblh = pblh.chunk('auto')\n",
    "\n",
    "# lat2 = xr.DataArray(df1['lat2'].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1['lon2'].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# pblh = pblh.sel(lat=lat2, lon=lon2, method='nearest')\n",
    "# pblh = pblh.compute()\n",
    "\n",
    "# pblh = pblh.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# pblh = pblh.drop_vars({'lon', 'lat'})\n",
    "\n",
    "# pblh.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_boundary_layer_height_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6efc839c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T11:14:50.994770Z",
     "start_time": "2022-11-10T11:14:50.991848Z"
    }
   },
   "source": [
    "### Surface net thermal radiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f6e23227",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T14:00:29.246588Z",
     "start_time": "2022-11-10T13:27:59.355013Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/bog/amuttaqin/Datasets/ERA5-Land/str/1991-2020_str.nc'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # Convert grib file to netcdf\n",
    "\n",
    "# filegrb = \"/bog/amuttaqin/Datasets/ERA5-Land/str/1991-2020_str.grib\"\n",
    "# filenc = \"/bog/amuttaqin/Datasets/ERA5-Land/str/1991-2020_str.nc\"\n",
    "\n",
    "# cdo.copy(input=filegrb, options='-f nc4c -z zip', output=filenc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1901a374",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T14:31:29.384571Z",
     "start_time": "2022-11-10T14:00:29.298536Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Extract variable at transects\n",
    "# sntr = xr.open_dataset('/bog/amuttaqin/Datasets/ERA5-Land/str/1991-2020_str.nc')\n",
    "# sntr = sntr.chunk('auto')\n",
    "\n",
    "# lat2 = xr.DataArray(df1['lat2'].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1['lon2'].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# sntr = sntr.sel(lat=lat2, lon=lon2, method='nearest')\n",
    "# sntr = sntr.compute()\n",
    "\n",
    "# sntr = sntr.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# sntr = sntr.drop_vars({'lon', 'lat'})\n",
    "\n",
    "# sntr.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_surface_net_thermal_radiation_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9670aaf6",
   "metadata": {},
   "source": [
    "### IMERG Precipitation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "797f0421",
   "metadata": {},
   "source": [
    "Combine monthly data to yearly using CDO Mergetime (2000-2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ac96fcf9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T15:43:46.686345Z",
     "start_time": "2022-11-10T15:03:02.158201Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/bog/amuttaqin/Datasets/IMERG-GPM/2000-2020/2000-2020_imerg_precip.nc4'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fileinputs = str(\"/bog/amuttaqin/Datasets/IMERG-GPM/2000-2020/*.nc4\")\n",
    "# fileoutput = str('/bog/amuttaqin/Datasets/IMERG-GPM/2000-2020/2000-2020_imerg_precip.nc4')\n",
    "\n",
    "# cdo.mergetime(input=glob.glob(fileinputs), output=fileoutput)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f18cc07d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-10T17:21:37.318648Z",
     "start_time": "2022-11-10T15:43:46.788072Z"
    }
   },
   "outputs": [],
   "source": [
    "# imerg = xr.open_dataset('/bog/amuttaqin/Datasets/IMERG-GPM/2000-2020/2000-2020_imerg_precip.nc4')\n",
    "# imerg = imerg.chunk('auto')\n",
    "\n",
    "# lat2 = xr.DataArray(df1['lat2'].to_numpy().tolist(), dims='segment')\n",
    "# lon2 = xr.DataArray(df1['lon2'].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "# imerg = imerg.sel(lat=lat2, lon=lon2, method='nearest')\n",
    "# imerg = imerg.compute()\n",
    "\n",
    "# imerg = imerg.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "# imerg = imerg.drop_vars({'lon','lat'})\n",
    "\n",
    "# imerg.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_imerg_precip_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1b26e55",
   "metadata": {},
   "source": [
    "Multilevel wind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bf7c5f64",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T01:47:19.242404Z",
     "start_time": "2022-11-10T22:03:02.914831Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/bog/amuttaqin/Datasets/ERA5/uv/uv_multilevel/1991-2020_uv_multilevel.nc'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # Convert grib file to netcdf\n",
    "\n",
    "# filegrb = \"/bog/amuttaqin/Datasets/ERA5/uv/uv_multilevel/1991-2020_uv_multilevel.grib\"\n",
    "# filenc = \"/bog/amuttaqin/Datasets/ERA5/uv/uv_multilevel/1991-2020_uv_multilevel.nc\"\n",
    "\n",
    "# cdo.copy(input=filegrb, options='-f nc4c -z zip', output=filenc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc3813ae",
   "metadata": {},
   "source": [
    "Evapotranspiration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "96c86f0e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T02:38:29.073034Z",
     "start_time": "2022-11-11T01:53:03.398312Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/bog/amuttaqin/Datasets/ERA5-Land/et/1991-2020_et.nc'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Convert grib file to netcdf\n",
    "\n",
    "filegrb = \"/bog/amuttaqin/Datasets/ERA5-Land/et/1991-2020_et.grib\"\n",
    "filenc  = \"/bog/amuttaqin/Datasets/ERA5-Land/et/1991-2020_et.nc\"\n",
    "\n",
    "cdo.copy(input=filegrb, options='-f nc4c -z zip', output=filenc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b9c736f7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T04:01:21.573056Z",
     "start_time": "2022-11-11T03:20:00.481439Z"
    }
   },
   "outputs": [],
   "source": [
    "## Extract variable at transects\n",
    "\n",
    "et = xr.open_dataset('/bog/amuttaqin/Datasets/ERA5-Land/et/1991-2020_et.nc')\n",
    "et = et.chunk('auto')\n",
    "\n",
    "lat2 = xr.DataArray(df1['lat2'].to_numpy().tolist(), dims='segment')\n",
    "lon2 = xr.DataArray(df1['lon2'].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "et = et.sel(lat=lat2, lon=lon2, method='nearest')\n",
    "et = et.compute()\n",
    "\n",
    "et = et.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "et = et.drop_vars({'lon', 'lat'})\n",
    "\n",
    "et.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_evapotranspiration_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3152a54d",
   "metadata": {},
   "source": [
    "Temperature at 2-m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3c0d42f5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T03:20:00.424156Z",
     "start_time": "2022-11-11T02:38:29.122965Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/bog/amuttaqin/Datasets/ERA5-Land/2t/1991-2020_2t.nc'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Convert grib file to netcdf\n",
    "\n",
    "filegrb = \"/bog/amuttaqin/Datasets/ERA5-Land/2t/1991-2020_2t.grib\"\n",
    "filenc  = \"/bog/amuttaqin/Datasets/ERA5-Land/2t/1991-2020_2t.nc\"\n",
    "\n",
    "cdo.copy(input=filegrb, options='-f nc4c -z zip', output=filenc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ea43f2ac",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-11T04:40:40.611254Z",
     "start_time": "2022-11-11T04:01:21.786714Z"
    }
   },
   "outputs": [],
   "source": [
    "## Extract variable at transects\n",
    "\n",
    "t2m = xr.open_dataset('/bog/amuttaqin/Datasets/ERA5-Land/2t/1991-2020_2t.nc')\n",
    "t2m = t2m.chunk('auto')\n",
    "\n",
    "lat2 = xr.DataArray(df1['lat2'].to_numpy().tolist(), dims='segment')\n",
    "lon2 = xr.DataArray(df1['lon2'].to_numpy().tolist(), dims='segment')\n",
    "\n",
    "t2m = t2m.sel(lat=lat2, lon=lon2, method='nearest')\n",
    "t2m = t2m.compute()\n",
    "\n",
    "t2m = t2m.assign_coords(segment=df1['segID'].to_numpy().tolist())\n",
    "t2m = t2m.drop_vars({'lon', 'lat'})\n",
    "\n",
    "t2m.to_netcdf('/bog/amuttaqin/Chapter2_SBC/ds_air_temp_2m_dc02.nc', 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39af5b92",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "288.849px",
    "left": "758.082px",
    "right": "20px",
    "top": "120px",
    "width": "331.009px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}