extracting_data_from_netcdf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6ec626ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "from netCDF4 import Dataset, num2date\n",
    "from datetime import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ca827b21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function to get the index of longitude and latitude\n",
    "def get_index(_val, _array):\n",
    "    return (np.abs(_array - _val)).argmin()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "efccd6b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the file containing centroid of the 21 pixels\n",
    "locations_df = pd.read_csv('station_07AA002.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1fa32949",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the netcdf files for all of Jan \n",
    "files = glob.glob('Jan1981/*.nc4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f87e5733",
   "metadata": {},
   "outputs": [],
   "source": [
    "variable_extracted = \"AvgSurfT_inst\" # variable to extract from netcdfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "af1e0365",
   "metadata": {},
   "outputs": [],
   "source": [
    "lats = locations_df.latitude.values\n",
    "lons = locations_df.longitude.values\n",
    "pixel_ids = locations_df.PIXEL_ID.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "705a64e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "hourly_dfs_list = []\n",
    "for _, file in enumerate(files):\n",
    "    nc_data = Dataset(file, 'r')\n",
    "    temp_pxl = []\n",
    "    temp_var = []\n",
    "    temp_hr = []\n",
    "    temp_yr = []\n",
    "    temp_mon = []\n",
    "    temp_day = []\n",
    "    for lat, lon, pixel_id in zip(lats, lons, pixel_ids):\n",
    "        times = num2date(nc_data.variables['time'][:], nc_data.variables['time'].units, nc_data.variables['time'].calendar)[:]\n",
    "        times_formatted = [dt(x.year,x.month,x.day,x.hour,x.minute) for x in times]\n",
    "        _hr = times_formatted[0].hour\n",
    "        _day = times_formatted[0].day\n",
    "        _yr = times_formatted[0].year\n",
    "        _mon = times_formatted[0].month\n",
    "        nc_lats = nc_data['lat'][:]\n",
    "        nc_lons = nc_data['lon'][:]\n",
    "        lat_index = get_index(lat, nc_lats)\n",
    "        lon_index = get_index(lon, nc_lons)\n",
    "        extracted_value = nc_data[variable_extracted][:, lat_index, lon_index].data[0]\n",
    "        \n",
    "        temp_var.append(extracted_value)\n",
    "        temp_pxl.append(pixel_id)\n",
    "        temp_hr.append(_hr)\n",
    "        temp_mon.append(_mon)\n",
    "        temp_yr.append(_yr)\n",
    "        temp_day.append(_day)\n",
    "    temp_df = pd.DataFrame()\n",
    "    temp_df['pixel_id'] = temp_pxl\n",
    "    temp_df['year'] = temp_yr\n",
    "    temp_df['month'] =temp_mon\n",
    "    temp_df['day'] = temp_day\n",
    "    temp_df['time'] = temp_hr\n",
    "    temp_df[variable_extracted] = temp_var\n",
    "    hourly_dfs_list.append(temp_df)\n",
    "merged_df = pd.concat(hourly_dfs_list, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3c29434e",
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df['Date'] = pd.to_datetime(merged_df[['year', 'month', 'day']])\n",
    "merged_df = merged_df.drop(['year', 'month', 'day', 'time', 'pixel_id'], axis=1)\n",
    "merged_df = merged_df.sort_values(by=[\"Date\"])\n",
    "merged_df = merged_df.groupby(['Date']).mean().reset_index()\n",
    "merged_df.to_csv('{}_Jan_1981_fullmonth_data.csv'.format(variable_extracted), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e70439bf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45f5cd0a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}