ETL_WDI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "56613a88",
   "metadata": {},
   "source": [
    "Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "72a6a846-e1c9-4961-80f1-07ec0213ccbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql.types import DecimalType\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b80ae3d1-a954-46b6-ac2a-e187b96b869f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Helper for pretty formatting for Spark DataFrames\n",
    "def showDF(df, limitRows = 20, truncate = True):\n",
    "    if(truncate):\n",
    "        pd.set_option('display.max_colwidth',50)\n",
    "        pd.reset_option('display.max_colwidth')      \n",
    "    else:\n",
    "        pd.set_option('display.max_colwidth',None)\n",
    "    pd.set_option('display.max_rows', limitRows)\n",
    "    display(df.limit(limitRows).toPandas())\n",
    "    pd.reset_option('display.max_rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea670f29-8356-4879-89f5-9d3c0f5be8e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a spark session\n",
    "spark = SparkSession.builder.appName(\"WDI Data Exploration\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9aa77053-c081-4589-ac61-3feeae172538",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Read the file \"WDICountry.csv\" into a Spark Data Frame\n",
    "df_country = spark.read.format(\"csv\") \\\n",
    "            .option(\"header\",\"true\") \\\n",
    "            .option(\"inferSchema\",\"true\") \\\n",
    "            .load(\"WDICountry.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7cbe01a9-c857-4f14-9956-675f4f3a725b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Country Code: string (nullable = true)\n",
      " |-- Short Name: string (nullable = true)\n",
      " |-- Table Name: string (nullable = true)\n",
      " |-- Long Name: string (nullable = true)\n",
      " |-- 2-alpha code: string (nullable = true)\n",
      " |-- Currency Unit: string (nullable = true)\n",
      " |-- Special Notes: string (nullable = true)\n",
      " |-- Region: string (nullable = true)\n",
      " |-- Income Group: string (nullable = true)\n",
      " |-- WB-2 code: string (nullable = true)\n",
      " |-- National accounts base year: string (nullable = true)\n",
      " |-- National accounts reference year: string (nullable = true)\n",
      " |-- SNA price valuation: string (nullable = true)\n",
      " |-- Lending category: string (nullable = true)\n",
      " |-- Other groups: string (nullable = true)\n",
      " |-- System of National Accounts: string (nullable = true)\n",
      " |-- Alternative conversion factor: string (nullable = true)\n",
      " |-- PPP survey year: string (nullable = true)\n",
      " |-- Balance of Payments Manual in use: string (nullable = true)\n",
      " |-- External debt Reporting status: string (nullable = true)\n",
      " |-- System of trade: string (nullable = true)\n",
      " |-- Government Accounting concept: string (nullable = true)\n",
      " |-- IMF data dissemination standard: string (nullable = true)\n",
      " |-- Latest population census: string (nullable = true)\n",
      " |-- Latest household survey: string (nullable = true)\n",
      " |-- Source of most recent Income and expenditure data: string (nullable = true)\n",
      " |-- Vital registration complete: string (nullable = true)\n",
      " |-- Latest agricultural census: string (nullable = true)\n",
      " |-- Latest industrial data: integer (nullable = true)\n",
      " |-- Latest trade data: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Inspect the schema\n",
    "df_country.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9c994a37-ac3e-4d09-9bf2-510cfb448508",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24/07/27 23:09:29 WARN SparkStringUtils: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country Code</th>\n",
       "      <th>Short Name</th>\n",
       "      <th>Table Name</th>\n",
       "      <th>Long Name</th>\n",
       "      <th>2-alpha code</th>\n",
       "      <th>Currency Unit</th>\n",
       "      <th>Special Notes</th>\n",
       "      <th>Region</th>\n",
       "      <th>Income Group</th>\n",
       "      <th>WB-2 code</th>\n",
       "      <th>...</th>\n",
       "      <th>System of trade</th>\n",
       "      <th>Government Accounting concept</th>\n",
       "      <th>IMF data dissemination standard</th>\n",
       "      <th>Latest population census</th>\n",
       "      <th>Latest household survey</th>\n",
       "      <th>Source of most recent Income and expenditure data</th>\n",
       "      <th>Vital registration complete</th>\n",
       "      <th>Latest agricultural census</th>\n",
       "      <th>Latest industrial data</th>\n",
       "      <th>Latest trade data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>AW</td>\n",
       "      <td>Aruban florin</td>\n",
       "      <td>None</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "      <td>AW</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>None</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2020 (expected)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>ZH</td>\n",
       "      <td>None</td>\n",
       "      <td>26 countries, stretching from the Red Sea in the North to the Cape of Good Hope in the South (https://www.worldbank.org/en/region/afr/eastern-and-southern-africa)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ZH</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Islamic State of Afghanistan</td>\n",
       "      <td>AF</td>\n",
       "      <td>Afghan afghani</td>\n",
       "      <td>The reporting period for national accounts data is designated as either calendar year basis (CY) or fiscal year basis (FY). For this country, it is fiscal year-based (fiscal year-end: March 20). Also, an estimate (PA.NUS.ATLS) of the exchange rate covers the same period and thus differs from the official exchange rate (CY).</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In addition</td>\n",
       "      <td>the World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country</td>\n",
       "      <td>multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country</td>\n",
       "      <td>this applies to the period 1960-2006. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.\"</td>\n",
       "      <td>South Asia</td>\n",
       "      <td>Low income</td>\n",
       "      <td>AF</td>\n",
       "      <td>2016</td>\n",
       "      <td>None</td>\n",
       "      <td>Value added at basic prices (VAB)</td>\n",
       "      <td>...</td>\n",
       "      <td>1979</td>\n",
       "      <td>Demographic and Health Survey, 2015</td>\n",
       "      <td>Integrated household survey (IHS), 2016/17</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>ZI</td>\n",
       "      <td>None</td>\n",
       "      <td>22 countries, stretching from the westernmost point of Africa, across the equator, and partly along the Atlantic Ocean till the Republic of Congo in the South (https://www.worldbank.org/en/region/afr/western-and-central-africa)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ZI</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>Angola</td>\n",
       "      <td>People's Republic of Angola</td>\n",
       "      <td>AO</td>\n",
       "      <td>Angolan kwanza</td>\n",
       "      <td>The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1994-2021. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "      <td>AO</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Budgetary central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2014</td>\n",
       "      <td>Demographic and Health Survey, 2015/16</td>\n",
       "      <td>Integrated household survey (IHS), 2008/09</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>Albania</td>\n",
       "      <td>Republic of Albania</td>\n",
       "      <td>AL</td>\n",
       "      <td>Albanian lek</td>\n",
       "      <td>None</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "      <td>AL</td>\n",
       "      <td>...</td>\n",
       "      <td>Special trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2020 (expected)</td>\n",
       "      <td>Demographic and Health Survey, 2017/18</td>\n",
       "      <td>Living Standards Measurement Study Survey (LSMS), 2012</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AND</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>Principality of Andorra</td>\n",
       "      <td>AD</td>\n",
       "      <td>Euro</td>\n",
       "      <td>None</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "      <td>AD</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2011. Population figures compiled from administrative registers.</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1A</td>\n",
       "      <td>None</td>\n",
       "      <td>Arab World aggregate. Arab World is composed of members of the League of Arab States.</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1A</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ARE</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>AE</td>\n",
       "      <td>U.A.E. dirham</td>\n",
       "      <td>None</td>\n",
       "      <td>Middle East &amp; North Africa</td>\n",
       "      <td>High income</td>\n",
       "      <td>AE</td>\n",
       "      <td>...</td>\n",
       "      <td>Special trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2020 (expected)</td>\n",
       "      <td>World Health Survey, 2003</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ARG</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>Argentine Republic</td>\n",
       "      <td>AR</td>\n",
       "      <td>Argentine peso</td>\n",
       "      <td>The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1971-2018. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>Upper middle income</td>\n",
       "      <td>AR</td>\n",
       "      <td>...</td>\n",
       "      <td>Special trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Special Data Dissemination Standard (SDDS)</td>\n",
       "      <td>2020 (expected)</td>\n",
       "      <td>Multiple Indicator Cluster Survey, 2019/20</td>\n",
       "      <td>Integrated household survey (IHS), 2016</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2008</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ARM</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>Republic of Armenia</td>\n",
       "      <td>AM</td>\n",
       "      <td>Armenian dram</td>\n",
       "      <td>None</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "      <td>AM</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Special Data Dissemination Standard (SDDS)</td>\n",
       "      <td>2020 (expected)</td>\n",
       "      <td>Demographic and Health Survey, 2015/16</td>\n",
       "      <td>Integrated household survey (IHS), 2016</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2014</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ASM</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>AS</td>\n",
       "      <td>U.S. dollar</td>\n",
       "      <td>None</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>Upper middle income</td>\n",
       "      <td>AS</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2020 (expected)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2008</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ATG</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>AG</td>\n",
       "      <td>East Caribbean dollar</td>\n",
       "      <td>None</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "      <td>AG</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Budgetary central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2011</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2007</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AUS</td>\n",
       "      <td>Australia</td>\n",
       "      <td>Australia</td>\n",
       "      <td>Commonwealth of Australia</td>\n",
       "      <td>AU</td>\n",
       "      <td>Australian dollar</td>\n",
       "      <td>The reporting period for national accounts data is designated as either calendar year basis (CY) or fiscal year basis (FY). For this country, it is fiscal year-based (fiscal year-end: June 30). Also, an estimate (PA.NUS.ATLS) of the exchange rate covers the same period and thus differs from the official exchange rate (CY).</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>High income</td>\n",
       "      <td>AU</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Special Data Dissemination Standard (SDDS)</td>\n",
       "      <td>2016</td>\n",
       "      <td>None</td>\n",
       "      <td>Expenditure survey/budget survey (ES/BS), 2010</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2015-2016</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AUT</td>\n",
       "      <td>Austria</td>\n",
       "      <td>Austria</td>\n",
       "      <td>Republic of Austria</td>\n",
       "      <td>AT</td>\n",
       "      <td>Euro</td>\n",
       "      <td>A simple multiplier is used to convert the national currencies of EMU members to euros. The following irrevocable euro conversion rate was adopted by the EU Council on January 1, 1999: 1 euro = 13.7603 Austrian schilling. Please note that historical data before 1999 are not actual euros and are not comparable or suitable for aggregation across countries.</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "      <td>AT</td>\n",
       "      <td>...</td>\n",
       "      <td>Special trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Special Data Dissemination Standard Plus (SDDS Plus)</td>\n",
       "      <td>2011. Population figures compiled from administrative registers.</td>\n",
       "      <td>None</td>\n",
       "      <td>Income survey (IS), 2015</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2010</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AZE</td>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Republic of Azerbaijan</td>\n",
       "      <td>AZ</td>\n",
       "      <td>New Azeri manat</td>\n",
       "      <td>None</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "      <td>AZ</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2019</td>\n",
       "      <td>Demographic and Health Survey, 2006</td>\n",
       "      <td>Living Standards Measurement Study Survey (LSMS), 2012</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2015</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BDI</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>Republic of Burundi</td>\n",
       "      <td>BI</td>\n",
       "      <td>Burundi franc</td>\n",
       "      <td>The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1983-2021. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Low income</td>\n",
       "      <td>BI</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Budgetary central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2008</td>\n",
       "      <td>Demographic and Health Survey, 2016/17</td>\n",
       "      <td>Core Welfare Indicator Questionnaire Survey (CWIQ), 2014</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>BEL</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>Kingdom of Belgium</td>\n",
       "      <td>BE</td>\n",
       "      <td>Euro</td>\n",
       "      <td>A simple multiplier is used to convert the national currencies of EMU members to euros. The following irrevocable euro conversion rate was adopted by the EU Council on January 1, 1999: 1 euro = 40.3399 Belgian franc. Please note that historical data before 1999 are not actual euros and are not comparable or suitable for aggregation across countries.</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "      <td>BE</td>\n",
       "      <td>...</td>\n",
       "      <td>Special trade system</td>\n",
       "      <td>Consolidated central government</td>\n",
       "      <td>Special Data Dissemination Standard (SDDS)</td>\n",
       "      <td>2011. Population figures compiled from administrative registers in combination with other sources of data, such as sample surveys.</td>\n",
       "      <td>None</td>\n",
       "      <td>Income survey (IS), 2015</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2010</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>BEN</td>\n",
       "      <td>Benin</td>\n",
       "      <td>Benin</td>\n",
       "      <td>Republic of Benin</td>\n",
       "      <td>BJ</td>\n",
       "      <td>West African CFA franc</td>\n",
       "      <td>None</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "      <td>BJ</td>\n",
       "      <td>...</td>\n",
       "      <td>General trade system</td>\n",
       "      <td>Budgetary central government</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-GDDS)</td>\n",
       "      <td>2013</td>\n",
       "      <td>Demographic and Health Survey, 2017/18</td>\n",
       "      <td>Core Welfare Indicator Questionnaire Survey (CWIQ), 2015</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1981.0</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Country Code  \\\n",
       "0           ABW   \n",
       "1           AFE   \n",
       "2           AFG   \n",
       "3   In addition   \n",
       "4           AFW   \n",
       "5           AGO   \n",
       "6           ALB   \n",
       "7           AND   \n",
       "8           ARB   \n",
       "9           ARE   \n",
       "10          ARG   \n",
       "11          ARM   \n",
       "12          ASM   \n",
       "13          ATG   \n",
       "14          AUS   \n",
       "15          AUT   \n",
       "16          AZE   \n",
       "17          BDI   \n",
       "18          BEL   \n",
       "19          BEN   \n",
       "\n",
       "                                                                                                                       Short Name  \\\n",
       "0                                                                                                                           Aruba   \n",
       "1                                                                                                     Africa Eastern and Southern   \n",
       "2                                                                                                                     Afghanistan   \n",
       "3    the World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country   \n",
       "4                                                                                                      Africa Western and Central   \n",
       "5                                                                                                                          Angola   \n",
       "6                                                                                                                         Albania   \n",
       "7                                                                                                                         Andorra   \n",
       "8                                                                                                                      Arab World   \n",
       "9                                                                                                            United Arab Emirates   \n",
       "10                                                                                                                      Argentina   \n",
       "11                                                                                                                        Armenia   \n",
       "12                                                                                                                 American Samoa   \n",
       "13                                                                                                            Antigua and Barbuda   \n",
       "14                                                                                                                      Australia   \n",
       "15                                                                                                                        Austria   \n",
       "16                                                                                                                     Azerbaijan   \n",
       "17                                                                                                                        Burundi   \n",
       "18                                                                                                                        Belgium   \n",
       "19                                                                                                                          Benin   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                Table Name  \\\n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                    Aruba   \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                              Africa Eastern and Southern   \n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                              Afghanistan   \n",
       "3    multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country   \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                               Africa Western and Central   \n",
       "5                                                                                                                                                                                                                                                                                                                                                                                                                                                   Angola   \n",
       "6                                                                                                                                                                                                                                                                                                                                                                                                                                                  Albania   \n",
       "7                                                                                                                                                                                                                                                                                                                                                                                                                                                  Andorra   \n",
       "8                                                                                                                                                                                                                                                                                                                                                                                                                                               Arab World   \n",
       "9                                                                                                                                                                                                                                                                                                                                                                                                                                     United Arab Emirates   \n",
       "10                                                                                                                                                                                                                                                                                                                                                                                                                                               Argentina   \n",
       "11                                                                                                                                                                                                                                                                                                                                                                                                                                                 Armenia   \n",
       "12                                                                                                                                                                                                                                                                                                                                                                                                                                          American Samoa   \n",
       "13                                                                                                                                                                                                                                                                                                                                                                                                                                     Antigua and Barbuda   \n",
       "14                                                                                                                                                                                                                                                                                                                                                                                                                                               Australia   \n",
       "15                                                                                                                                                                                                                                                                                                                                                                                                                                                 Austria   \n",
       "16                                                                                                                                                                                                                                                                                                                                                                                                                                              Azerbaijan   \n",
       "17                                                                                                                                                                                                                                                                                                                                                                                                                                                 Burundi   \n",
       "18                                                                                                                                                                                                                                                                                                                                                                                                                                                 Belgium   \n",
       "19                                                                                                                                                                                                                                                                                                                                                                                                                                                   Benin   \n",
       "\n",
       "                                                                                                                                                                                    Long Name  \\\n",
       "0                                                                                                                                                                                       Aruba   \n",
       "1                                                                                                                                                                 Africa Eastern and Southern   \n",
       "2                                                                                                                                                                Islamic State of Afghanistan   \n",
       "3    this applies to the period 1960-2006. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.\"   \n",
       "4                                                                                                                                                                  Africa Western and Central   \n",
       "5                                                                                                                                                                 People's Republic of Angola   \n",
       "6                                                                                                                                                                         Republic of Albania   \n",
       "7                                                                                                                                                                     Principality of Andorra   \n",
       "8                                                                                                                                                                                  Arab World   \n",
       "9                                                                                                                                                                        United Arab Emirates   \n",
       "10                                                                                                                                                                         Argentine Republic   \n",
       "11                                                                                                                                                                        Republic of Armenia   \n",
       "12                                                                                                                                                                             American Samoa   \n",
       "13                                                                                                                                                                        Antigua and Barbuda   \n",
       "14                                                                                                                                                                  Commonwealth of Australia   \n",
       "15                                                                                                                                                                        Republic of Austria   \n",
       "16                                                                                                                                                                     Republic of Azerbaijan   \n",
       "17                                                                                                                                                                        Republic of Burundi   \n",
       "18                                                                                                                                                                         Kingdom of Belgium   \n",
       "19                                                                                                                                                                          Republic of Benin   \n",
       "\n",
       "   2-alpha code           Currency Unit  \\\n",
       "0            AW           Aruban florin   \n",
       "1            ZH                    None   \n",
       "2            AF          Afghan afghani   \n",
       "3    South Asia              Low income   \n",
       "4            ZI                    None   \n",
       "5            AO          Angolan kwanza   \n",
       "6            AL            Albanian lek   \n",
       "7            AD                    Euro   \n",
       "8            1A                    None   \n",
       "9            AE           U.A.E. dirham   \n",
       "10           AR          Argentine peso   \n",
       "11           AM           Armenian dram   \n",
       "12           AS             U.S. dollar   \n",
       "13           AG   East Caribbean dollar   \n",
       "14           AU       Australian dollar   \n",
       "15           AT                    Euro   \n",
       "16           AZ         New Azeri manat   \n",
       "17           BI           Burundi franc   \n",
       "18           BE                    Euro   \n",
       "19           BJ  West African CFA franc   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   Special Notes  \\\n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           None   \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             26 countries, stretching from the Red Sea in the North to the Cape of Good Hope in the South (https://www.worldbank.org/en/region/afr/eastern-and-southern-africa)   \n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                          The reporting period for national accounts data is designated as either calendar year basis (CY) or fiscal year basis (FY). For this country, it is fiscal year-based (fiscal year-end: March 20). Also, an estimate (PA.NUS.ATLS) of the exchange rate covers the same period and thus differs from the official exchange rate (CY).   \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             AF   \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            22 countries, stretching from the westernmost point of Africa, across the equator, and partly along the Atlantic Ocean till the Republic of Congo in the South (https://www.worldbank.org/en/region/afr/western-and-central-africa)   \n",
       "5   The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1994-2021. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.   \n",
       "6                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           None   \n",
       "7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           None   \n",
       "8                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          Arab World aggregate. Arab World is composed of members of the League of Arab States.   \n",
       "9                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           None   \n",
       "10  The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1971-2018. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.   \n",
       "11                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          None   \n",
       "12                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          None   \n",
       "13                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          None   \n",
       "14                                                                                                                                                                                                                                                                                                                                                                                                                                          The reporting period for national accounts data is designated as either calendar year basis (CY) or fiscal year basis (FY). For this country, it is fiscal year-based (fiscal year-end: June 30). Also, an estimate (PA.NUS.ATLS) of the exchange rate covers the same period and thus differs from the official exchange rate (CY).   \n",
       "15                                                                                                                                                                                                                                                                                                                                                                                                          A simple multiplier is used to convert the national currencies of EMU members to euros. The following irrevocable euro conversion rate was adopted by the EU Council on January 1, 1999: 1 euro = 13.7603 Austrian schilling. Please note that historical data before 1999 are not actual euros and are not comparable or suitable for aggregation across countries.   \n",
       "16                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          None   \n",
       "17  The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1983-2021. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.   \n",
       "18                                                                                                                                                                                                                                                                                                                                                                                                               A simple multiplier is used to convert the national currencies of EMU members to euros. The following irrevocable euro conversion rate was adopted by the EU Council on January 1, 1999: 1 euro = 40.3399 Belgian franc. Please note that historical data before 1999 are not actual euros and are not comparable or suitable for aggregation across countries.   \n",
       "19                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          None   \n",
       "\n",
       "                        Region         Income Group  \\\n",
       "0    Latin America & Caribbean          High income   \n",
       "1                         None                 None   \n",
       "2                         None                 None   \n",
       "3                         2016                 None   \n",
       "4                         None                 None   \n",
       "5           Sub-Saharan Africa  Lower middle income   \n",
       "6        Europe & Central Asia  Upper middle income   \n",
       "7        Europe & Central Asia          High income   \n",
       "8                         None                 None   \n",
       "9   Middle East & North Africa          High income   \n",
       "10   Latin America & Caribbean  Upper middle income   \n",
       "11       Europe & Central Asia  Upper middle income   \n",
       "12         East Asia & Pacific  Upper middle income   \n",
       "13   Latin America & Caribbean          High income   \n",
       "14         East Asia & Pacific          High income   \n",
       "15       Europe & Central Asia          High income   \n",
       "16       Europe & Central Asia  Upper middle income   \n",
       "17          Sub-Saharan Africa           Low income   \n",
       "18       Europe & Central Asia          High income   \n",
       "19          Sub-Saharan Africa  Lower middle income   \n",
       "\n",
       "                            WB-2 code  ...       System of trade  \\\n",
       "0                                  AW  ...  General trade system   \n",
       "1                                  ZH  ...                  None   \n",
       "2                                None  ...                  None   \n",
       "3   Value added at basic prices (VAB)  ...                  1979   \n",
       "4                                  ZI  ...                  None   \n",
       "5                                  AO  ...  General trade system   \n",
       "6                                  AL  ...  Special trade system   \n",
       "7                                  AD  ...  General trade system   \n",
       "8                                  1A  ...                  None   \n",
       "9                                  AE  ...  Special trade system   \n",
       "10                                 AR  ...  Special trade system   \n",
       "11                                 AM  ...  General trade system   \n",
       "12                                 AS  ...                  None   \n",
       "13                                 AG  ...  General trade system   \n",
       "14                                 AU  ...  General trade system   \n",
       "15                                 AT  ...  Special trade system   \n",
       "16                                 AZ  ...  General trade system   \n",
       "17                                 BI  ...  General trade system   \n",
       "18                                 BE  ...  Special trade system   \n",
       "19                                 BJ  ...  General trade system   \n",
       "\n",
       "          Government Accounting concept  \\\n",
       "0                                  None   \n",
       "1                                  None   \n",
       "2                                  None   \n",
       "3   Demographic and Health Survey, 2015   \n",
       "4                                  None   \n",
       "5          Budgetary central government   \n",
       "6       Consolidated central government   \n",
       "7                                  None   \n",
       "8                                  None   \n",
       "9       Consolidated central government   \n",
       "10      Consolidated central government   \n",
       "11      Consolidated central government   \n",
       "12                                 None   \n",
       "13         Budgetary central government   \n",
       "14      Consolidated central government   \n",
       "15      Consolidated central government   \n",
       "16      Consolidated central government   \n",
       "17         Budgetary central government   \n",
       "18      Consolidated central government   \n",
       "19         Budgetary central government   \n",
       "\n",
       "                         IMF data dissemination standard  \\\n",
       "0    Enhanced General Data Dissemination System (e-GDDS)   \n",
       "1                                                   None   \n",
       "2                                                   None   \n",
       "3             Integrated household survey (IHS), 2016/17   \n",
       "4                                                   None   \n",
       "5    Enhanced General Data Dissemination System (e-GDDS)   \n",
       "6    Enhanced General Data Dissemination System (e-GDDS)   \n",
       "7                                                   None   \n",
       "8                                                   None   \n",
       "9    Enhanced General Data Dissemination System (e-GDDS)   \n",
       "10            Special Data Dissemination Standard (SDDS)   \n",
       "11            Special Data Dissemination Standard (SDDS)   \n",
       "12                                                  None   \n",
       "13   Enhanced General Data Dissemination System (e-GDDS)   \n",
       "14            Special Data Dissemination Standard (SDDS)   \n",
       "15  Special Data Dissemination Standard Plus (SDDS Plus)   \n",
       "16   Enhanced General Data Dissemination System (e-GDDS)   \n",
       "17   Enhanced General Data Dissemination System (e-GDDS)   \n",
       "18            Special Data Dissemination Standard (SDDS)   \n",
       "19   Enhanced General Data Dissemination System (e-GDDS)   \n",
       "\n",
       "                                                                                                              Latest population census  \\\n",
       "0                                                                                                                      2020 (expected)   \n",
       "1                                                                                                                                 None   \n",
       "2                                                                                                                                 None   \n",
       "3                                                                                                                                 None   \n",
       "4                                                                                                                                 None   \n",
       "5                                                                                                                                 2014   \n",
       "6                                                                                                                      2020 (expected)   \n",
       "7                                                                     2011. Population figures compiled from administrative registers.   \n",
       "8                                                                                                                                 None   \n",
       "9                                                                                                                      2020 (expected)   \n",
       "10                                                                                                                     2020 (expected)   \n",
       "11                                                                                                                     2020 (expected)   \n",
       "12                                                                                                                     2020 (expected)   \n",
       "13                                                                                                                                2011   \n",
       "14                                                                                                                                2016   \n",
       "15                                                                    2011. Population figures compiled from administrative registers.   \n",
       "16                                                                                                                                2019   \n",
       "17                                                                                                                                2008   \n",
       "18  2011. Population figures compiled from administrative registers in combination with other sources of data, such as sample surveys.   \n",
       "19                                                                                                                                2013   \n",
       "\n",
       "                       Latest household survey  \\\n",
       "0                                         None   \n",
       "1                                         None   \n",
       "2                                         None   \n",
       "3                                         None   \n",
       "4                                         None   \n",
       "5       Demographic and Health Survey, 2015/16   \n",
       "6       Demographic and Health Survey, 2017/18   \n",
       "7                                         None   \n",
       "8                                         None   \n",
       "9                    World Health Survey, 2003   \n",
       "10  Multiple Indicator Cluster Survey, 2019/20   \n",
       "11      Demographic and Health Survey, 2015/16   \n",
       "12                                        None   \n",
       "13                                        None   \n",
       "14                                        None   \n",
       "15                                        None   \n",
       "16         Demographic and Health Survey, 2006   \n",
       "17      Demographic and Health Survey, 2016/17   \n",
       "18                                        None   \n",
       "19      Demographic and Health Survey, 2017/18   \n",
       "\n",
       "           Source of most recent Income and expenditure data  \\\n",
       "0                                                       None   \n",
       "1                                                       None   \n",
       "2                                                       None   \n",
       "3                                                       None   \n",
       "4                                                       None   \n",
       "5                 Integrated household survey (IHS), 2008/09   \n",
       "6     Living Standards Measurement Study Survey (LSMS), 2012   \n",
       "7                                                       None   \n",
       "8                                                       None   \n",
       "9                                                       None   \n",
       "10                   Integrated household survey (IHS), 2016   \n",
       "11                   Integrated household survey (IHS), 2016   \n",
       "12                                                      None   \n",
       "13                                                      None   \n",
       "14            Expenditure survey/budget survey (ES/BS), 2010   \n",
       "15                                  Income survey (IS), 2015   \n",
       "16    Living Standards Measurement Study Survey (LSMS), 2012   \n",
       "17  Core Welfare Indicator Questionnaire Survey (CWIQ), 2014   \n",
       "18                                  Income survey (IS), 2015   \n",
       "19  Core Welfare Indicator Questionnaire Survey (CWIQ), 2015   \n",
       "\n",
       "   Vital registration complete Latest agricultural census  \\\n",
       "0                          Yes                       None   \n",
       "1                         None                       None   \n",
       "2                         None                       None   \n",
       "3                         2018                       None   \n",
       "4                         None                       None   \n",
       "5                         None                       None   \n",
       "6                          Yes                       2012   \n",
       "7                          Yes                       None   \n",
       "8                         None                       None   \n",
       "9                          Yes                       2012   \n",
       "10                         Yes                       2008   \n",
       "11                         Yes                       2014   \n",
       "12                         Yes                       2008   \n",
       "13                         Yes                       2007   \n",
       "14                         Yes                  2015-2016   \n",
       "15                         Yes                       2010   \n",
       "16                         Yes                       2015   \n",
       "17                        None                       None   \n",
       "18                         Yes                       2010   \n",
       "19                        None                       None   \n",
       "\n",
       "   Latest industrial data Latest trade data  \n",
       "0                     NaN            2018.0  \n",
       "1                     NaN               NaN  \n",
       "2                     NaN               NaN  \n",
       "3                     NaN               NaN  \n",
       "4                     NaN               NaN  \n",
       "5                     NaN            2018.0  \n",
       "6                  2013.0            2018.0  \n",
       "7                     NaN            2018.0  \n",
       "8                     NaN               NaN  \n",
       "9                  1985.0            2018.0  \n",
       "10                 2002.0            2018.0  \n",
       "11                    NaN            2018.0  \n",
       "12                    NaN               NaN  \n",
       "13                    NaN            2018.0  \n",
       "14                 2013.0            2018.0  \n",
       "15                 2014.0            2018.0  \n",
       "16                 2013.0            2018.0  \n",
       "17                 2012.0            2018.0  \n",
       "18                 2014.0            2018.0  \n",
       "19                 1981.0            2018.0  \n",
       "\n",
       "[20 rows x 30 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the helper function to display the DataFrame with all columns and content fully visible\n",
    "showDF(df_country, truncate = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "73d230b2-d964-4896-807d-180c075a7b97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "278"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get some basic stats\n",
    "df_country.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd921c6f",
   "metadata": {},
   "source": [
    "Examining Dimensions\n",
    "\n",
    "How many different regions do the various countries belong to ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74833613",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>South Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016/2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016/17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>North America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Middle East &amp; North Africa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Region\n",
       "0                   South Asia\n",
       "1                         2016\n",
       "2                    2016/2017\n",
       "3                      2016/17\n",
       "4           Sub-Saharan Africa\n",
       "5        Europe & Central Asia\n",
       "6                North America\n",
       "7          East Asia & Pacific\n",
       "8   Middle East & North Africa\n",
       "9    Latin America & Caribbean\n",
       "10                        None"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showDF(df_country.select('Region').distinct(), truncate = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d2daad8",
   "metadata": {},
   "source": [
    "How many different income groups do we have across countries?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6c36424c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Income Group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Value added at producer prices (VAP)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Value added at basic prices (VAB)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Low income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Income Group\n",
       "0                   Lower middle income\n",
       "1  Value added at producer prices (VAP)\n",
       "2     Value added at basic prices (VAB)\n",
       "3                           High income\n",
       "4                   Upper middle income\n",
       "5                            Low income\n",
       "6                                  None"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showDF(df_country.select('Income Group').distinct(), truncate = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "76d5b635",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read the file \"WDISeries.csv\" into a Spark Data Frame\n",
    "df_series = spark.read.format(\"csv\") \\\n",
    "        .option(\"header\", \"true\") \\\n",
    "        .option(\"inferSchema\", \"true\") \\\n",
    "        .load(\"WDISeries.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e6a6a550",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Series Code: string (nullable = true)\n",
      " |-- Topic: string (nullable = true)\n",
      " |-- Indicator Name: string (nullable = true)\n",
      " |-- Short definition: string (nullable = true)\n",
      " |-- Long definition: string (nullable = true)\n",
      " |-- Unit of measure: string (nullable = true)\n",
      " |-- Periodicity: string (nullable = true)\n",
      " |-- Base Period: string (nullable = true)\n",
      " |-- Other notes: string (nullable = true)\n",
      " |-- Aggregation method: string (nullable = true)\n",
      " |-- Limitations and exceptions: string (nullable = true)\n",
      " |-- Notes from original source: string (nullable = true)\n",
      " |-- General comments: string (nullable = true)\n",
      " |-- Source: string (nullable = true)\n",
      " |-- Statistical concept and methodology: string (nullable = true)\n",
      " |-- Development relevance: string (nullable = true)\n",
      " |-- Related source links: string (nullable = true)\n",
      " |-- Other web links: string (nullable = true)\n",
      " |-- Related indicators: string (nullable = true)\n",
      " |-- License Type: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Inspect the schema\n",
    "df_series.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d69596ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Series Code</th>\n",
       "      <th>Topic</th>\n",
       "      <th>Indicator Name</th>\n",
       "      <th>Short definition</th>\n",
       "      <th>Long definition</th>\n",
       "      <th>Unit of measure</th>\n",
       "      <th>Periodicity</th>\n",
       "      <th>Base Period</th>\n",
       "      <th>Other notes</th>\n",
       "      <th>Aggregation method</th>\n",
       "      <th>Limitations and exceptions</th>\n",
       "      <th>Notes from original source</th>\n",
       "      <th>General comments</th>\n",
       "      <th>Source</th>\n",
       "      <th>Statistical concept and methodology</th>\n",
       "      <th>Development relevance</th>\n",
       "      <th>Related source links</th>\n",
       "      <th>Other web links</th>\n",
       "      <th>Related indicators</th>\n",
       "      <th>License Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AG.AGR.TRAC.NO</td>\n",
       "      <td>Environment: Agricultural production</td>\n",
       "      <td>Agricultural machinery, tractors</td>\n",
       "      <td>None</td>\n",
       "      <td>Agricultural machinery refers to the number of...</td>\n",
       "      <td>None</td>\n",
       "      <td>Annual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Sum</td>\n",
       "      <td>The data are collected by the Food and Agricul...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Food and Agriculture Organization, electronic ...</td>\n",
       "      <td>\"A tractor provides the power and traction to ...</td>\n",
       "      <td>for plowing</td>\n",
       "      <td>tilling</td>\n",
       "      <td>disking</td>\n",
       "      <td>harrowing</td>\n",
       "      <td>planting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A substantial contribution to agriculture in t...</td>\n",
       "      <td>steel plows</td>\n",
       "      <td>mowers</td>\n",
       "      <td>mechanical reapers</td>\n",
       "      <td>seed drills</td>\n",
       "      <td>and threshers contributed to the development ...</td>\n",
       "      <td>tractors enabled the farmer to sow and harves...</td>\n",
       "      <td>powered machinery such as tractors</td>\n",
       "      <td>has replaced many jobs formerly carried out b...</td>\n",
       "      <td>horses and mules. FAO estimates that most far...</td>\n",
       "      <td>seeds or agrochemicals.</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Agriculture is still a major sector in many ec...</td>\n",
       "      <td>and agricultural activities provide developin...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>There is no single correct mix of inputs to th...</td>\n",
       "      <td>as it is dependent on local climate</td>\n",
       "      <td>land quality</td>\n",
       "      <td>and economic development; appropriate levels ...</td>\n",
       "      <td>the climate and soils</td>\n",
       "      <td>and the production process used.\"</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>CC BY-4.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AG.CON.FERT.PT.ZS</td>\n",
       "      <td>Environment: Agricultural production</td>\n",
       "      <td>Fertilizer consumption (% of fertilizer produc...</td>\n",
       "      <td>None</td>\n",
       "      <td>Fertilizer consumption measures the quantity o...</td>\n",
       "      <td>None</td>\n",
       "      <td>Annual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Weighted average</td>\n",
       "      <td>The FAO has revised the time series for fertil...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The data are collected by the Food and Agricul...</td>\n",
       "      <td>but complete consistency across countries and...</td>\n",
       "      <td>national publications and related country dat...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Food and Agriculture Organization, electronic ...</td>\n",
       "      <td>Fertilizer consumption measures the quantity o...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Most fertilizers that are commonly used in agr...</td>\n",
       "      <td>phosphorus</td>\n",
       "      <td>and potassium. Some fertilizers also contain ...</td>\n",
       "      <td>\"\" such as zinc and other metals that are nece...</td>\n",
       "      <td>composts and other organic matter</td>\n",
       "      <td>and wastes</td>\n",
       "      <td>such as sewage sludge and certain industrial ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FAO defines arable land as land under temporar...</td>\n",
       "      <td>temporary meadows for mowing or for pasture</td>\n",
       "      <td>land under market or kitchen gardens</td>\n",
       "      <td>and land temporarily fallow; land abandoned a...</td>\n",
       "      <td>Factors such as the green revolution, has led ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Agriculture is still a major sector in many ec...</td>\n",
       "      <td>and agricultural activities provide developin...</td>\n",
       "      <td>pesticides</td>\n",
       "      <td>and intensive irrigation have environmental c...</td>\n",
       "      <td>inappropriate use of inputs for agricultural ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>In many developed countries</td>\n",
       "      <td>excessive nitrogen fertilizer applications ha...</td>\n",
       "      <td>longevity and overall fitness of certain agri...</td>\n",
       "      <td>such as aphids. Further</td>\n",
       "      <td>excessive use of fertilizers emits significan...</td>\n",
       "      <td>as \"\"fertilizer burn\"\" can occur when too muc...</td>\n",
       "      <td>resulting in drying out of the leaves and dam...</td>\n",
       "      <td>overuse of fertilizers has resulted in contam...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>There is no single correct mix of inputs to th...</td>\n",
       "      <td>as it is dependent on local climate</td>\n",
       "      <td>land quality</td>\n",
       "      <td>and economic development; appropriate levels ...</td>\n",
       "      <td>the climate and soils</td>\n",
       "      <td>and the production process used.\"</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>CC BY-4.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AG.CON.FERT.ZS</td>\n",
       "      <td>Environment: Agricultural production</td>\n",
       "      <td>Fertilizer consumption (kilograms per hectare ...</td>\n",
       "      <td>None</td>\n",
       "      <td>Fertilizer consumption measures the quantity o...</td>\n",
       "      <td>None</td>\n",
       "      <td>Annual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Weighted average</td>\n",
       "      <td>The FAO has revised the time series for fertil...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>The data are collected by the Food and Agricul...</td>\n",
       "      <td>but complete consistency across countries and...</td>\n",
       "      <td>national publications and related country dat...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Food and Agriculture Organization, electronic ...</td>\n",
       "      <td>Fertilizer consumption measures the quantity o...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Most fertilizers that are commonly used in agr...</td>\n",
       "      <td>phosphorus</td>\n",
       "      <td>and potassium. Some fertilizers also contain ...</td>\n",
       "      <td>\"\" such as zinc and other metals that are nece...</td>\n",
       "      <td>composts and other organic matter</td>\n",
       "      <td>and wastes</td>\n",
       "      <td>such as sewage sludge and certain industrial ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>FAO defines arable land as land under temporar...</td>\n",
       "      <td>temporary meadows for mowing or for pasture</td>\n",
       "      <td>land under market or kitchen gardens</td>\n",
       "      <td>and land temporarily fallow; land abandoned a...</td>\n",
       "      <td>Factors such as the green revolution, has led ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Agriculture is still a major sector in many ec...</td>\n",
       "      <td>and agricultural activities provide developin...</td>\n",
       "      <td>pesticides</td>\n",
       "      <td>and intensive irrigation have environmental c...</td>\n",
       "      <td>inappropriate use of inputs for agricultural ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>In many developed countries</td>\n",
       "      <td>excessive nitrogen fertilizer applications ha...</td>\n",
       "      <td>longevity and overall fitness of certain agri...</td>\n",
       "      <td>such as aphids. Further</td>\n",
       "      <td>excessive use of fertilizers emits significan...</td>\n",
       "      <td>as \"\"fertilizer burn\"\" can occur when too muc...</td>\n",
       "      <td>resulting in drying out of the leaves and dam...</td>\n",
       "      <td>overuse of fertilizers has resulted in contam...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>There is no single correct mix of inputs to th...</td>\n",
       "      <td>as it is dependent on local climate</td>\n",
       "      <td>land quality</td>\n",
       "      <td>and economic development; appropriate levels ...</td>\n",
       "      <td>the climate and soils</td>\n",
       "      <td>and the production process used.\"</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>CC BY-4.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AG.LND.AGRI.K2</td>\n",
       "      <td>Environment: Land use</td>\n",
       "      <td>Agricultural land (sq. km)</td>\n",
       "      <td>None</td>\n",
       "      <td>Agricultural land refers to the share of land ...</td>\n",
       "      <td>None</td>\n",
       "      <td>Annual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Sum</td>\n",
       "      <td>The data are collected by the Food and Agricul...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Food and Agriculture Organization, electronic ...</td>\n",
       "      <td>Agricultural land constitutes only a part of a...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Agricultural land is also sometimes classified...</td>\n",
       "      <td>with very little farming possible in non-irri...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Series Code  \\\n",
       "0                                      AG.AGR.TRAC.NO   \n",
       "1   A substantial contribution to agriculture in t...   \n",
       "2   Agriculture is still a major sector in many ec...   \n",
       "3   There is no single correct mix of inputs to th...   \n",
       "4                                   AG.CON.FERT.PT.ZS   \n",
       "5   The data are collected by the Food and Agricul...   \n",
       "6   Most fertilizers that are commonly used in agr...   \n",
       "7   FAO defines arable land as land under temporar...   \n",
       "8   Agriculture is still a major sector in many ec...   \n",
       "9                         In many developed countries   \n",
       "10  There is no single correct mix of inputs to th...   \n",
       "11                                     AG.CON.FERT.ZS   \n",
       "12  The data are collected by the Food and Agricul...   \n",
       "13  Most fertilizers that are commonly used in agr...   \n",
       "14  FAO defines arable land as land under temporar...   \n",
       "15  Agriculture is still a major sector in many ec...   \n",
       "16                        In many developed countries   \n",
       "17  There is no single correct mix of inputs to th...   \n",
       "18                                     AG.LND.AGRI.K2   \n",
       "19  Agricultural land is also sometimes classified...   \n",
       "\n",
       "                                                Topic  \\\n",
       "0                Environment: Agricultural production   \n",
       "1                                         steel plows   \n",
       "2    and agricultural activities provide developin...   \n",
       "3                 as it is dependent on local climate   \n",
       "4                Environment: Agricultural production   \n",
       "5    but complete consistency across countries and...   \n",
       "6                                          phosphorus   \n",
       "7         temporary meadows for mowing or for pasture   \n",
       "8    and agricultural activities provide developin...   \n",
       "9    excessive nitrogen fertilizer applications ha...   \n",
       "10                as it is dependent on local climate   \n",
       "11               Environment: Agricultural production   \n",
       "12   but complete consistency across countries and...   \n",
       "13                                         phosphorus   \n",
       "14        temporary meadows for mowing or for pasture   \n",
       "15   and agricultural activities provide developin...   \n",
       "16   excessive nitrogen fertilizer applications ha...   \n",
       "17                as it is dependent on local climate   \n",
       "18                              Environment: Land use   \n",
       "19   with very little farming possible in non-irri...   \n",
       "\n",
       "                                       Indicator Name  \\\n",
       "0                    Agricultural machinery, tractors   \n",
       "1                                              mowers   \n",
       "2                                                None   \n",
       "3                                        land quality   \n",
       "4   Fertilizer consumption (% of fertilizer produc...   \n",
       "5    national publications and related country dat...   \n",
       "6    and potassium. Some fertilizers also contain ...   \n",
       "7                land under market or kitchen gardens   \n",
       "8                                          pesticides   \n",
       "9    longevity and overall fitness of certain agri...   \n",
       "10                                       land quality   \n",
       "11  Fertilizer consumption (kilograms per hectare ...   \n",
       "12   national publications and related country dat...   \n",
       "13   and potassium. Some fertilizers also contain ...   \n",
       "14               land under market or kitchen gardens   \n",
       "15                                         pesticides   \n",
       "16   longevity and overall fitness of certain agri...   \n",
       "17                                       land quality   \n",
       "18                         Agricultural land (sq. km)   \n",
       "19                                               None   \n",
       "\n",
       "                                     Short definition  \\\n",
       "0                                                None   \n",
       "1                                  mechanical reapers   \n",
       "2                                                None   \n",
       "3    and economic development; appropriate levels ...   \n",
       "4                                                None   \n",
       "5                                                None   \n",
       "6   \"\" such as zinc and other metals that are nece...   \n",
       "7    and land temporarily fallow; land abandoned a...   \n",
       "8    and intensive irrigation have environmental c...   \n",
       "9                             such as aphids. Further   \n",
       "10   and economic development; appropriate levels ...   \n",
       "11                                               None   \n",
       "12                                               None   \n",
       "13  \"\" such as zinc and other metals that are nece...   \n",
       "14   and land temporarily fallow; land abandoned a...   \n",
       "15   and intensive irrigation have environmental c...   \n",
       "16                            such as aphids. Further   \n",
       "17   and economic development; appropriate levels ...   \n",
       "18                                               None   \n",
       "19                                               None   \n",
       "\n",
       "                                      Long definition  \\\n",
       "0   Agricultural machinery refers to the number of...   \n",
       "1                                         seed drills   \n",
       "2                                                None   \n",
       "3                               the climate and soils   \n",
       "4   Fertilizer consumption measures the quantity o...   \n",
       "5                                                None   \n",
       "6                   composts and other organic matter   \n",
       "7   Factors such as the green revolution, has led ...   \n",
       "8    inappropriate use of inputs for agricultural ...   \n",
       "9    excessive use of fertilizers emits significan...   \n",
       "10                              the climate and soils   \n",
       "11  Fertilizer consumption measures the quantity o...   \n",
       "12                                               None   \n",
       "13                  composts and other organic matter   \n",
       "14  Factors such as the green revolution, has led ...   \n",
       "15   inappropriate use of inputs for agricultural ...   \n",
       "16   excessive use of fertilizers emits significan...   \n",
       "17                              the climate and soils   \n",
       "18  Agricultural land refers to the share of land ...   \n",
       "19                                               None   \n",
       "\n",
       "                                      Unit of measure  \\\n",
       "0                                                None   \n",
       "1    and threshers contributed to the development ...   \n",
       "2                                                None   \n",
       "3                   and the production process used.\"   \n",
       "4                                                None   \n",
       "5   Food and Agriculture Organization, electronic ...   \n",
       "6                                          and wastes   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9    as \"\"fertilizer burn\"\" can occur when too muc...   \n",
       "10                  and the production process used.\"   \n",
       "11                                               None   \n",
       "12  Food and Agriculture Organization, electronic ...   \n",
       "13                                         and wastes   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16   as \"\"fertilizer burn\"\" can occur when too muc...   \n",
       "17                  and the production process used.\"   \n",
       "18                                               None   \n",
       "19                                               None   \n",
       "\n",
       "                                          Periodicity  \\\n",
       "0                                              Annual   \n",
       "1    tractors enabled the farmer to sow and harves...   \n",
       "2                                                None   \n",
       "3                                                None   \n",
       "4                                              Annual   \n",
       "5   Fertilizer consumption measures the quantity o...   \n",
       "6    such as sewage sludge and certain industrial ...   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9    resulting in drying out of the leaves and dam...   \n",
       "10                                               None   \n",
       "11                                             Annual   \n",
       "12  Fertilizer consumption measures the quantity o...   \n",
       "13   such as sewage sludge and certain industrial ...   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16   resulting in drying out of the leaves and dam...   \n",
       "17                                               None   \n",
       "18                                             Annual   \n",
       "19                                               None   \n",
       "\n",
       "                                          Base Period  \\\n",
       "0                                                None   \n",
       "1                  powered machinery such as tractors   \n",
       "2                                                None   \n",
       "3                                                None   \n",
       "4                                                None   \n",
       "5                                                None   \n",
       "6                                                None   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9    overuse of fertilizers has resulted in contam...   \n",
       "10                                               None   \n",
       "11                                               None   \n",
       "12                                               None   \n",
       "13                                               None   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16   overuse of fertilizers has resulted in contam...   \n",
       "17                                               None   \n",
       "18                                               None   \n",
       "19                                               None   \n",
       "\n",
       "                                          Other notes  \\\n",
       "0                                                None   \n",
       "1    has replaced many jobs formerly carried out b...   \n",
       "2                                                None   \n",
       "3                                                None   \n",
       "4                                                None   \n",
       "5                                                None   \n",
       "6                                                None   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9                                                None   \n",
       "10                                               None   \n",
       "11                                               None   \n",
       "12                                               None   \n",
       "13                                               None   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16                                               None   \n",
       "17                                               None   \n",
       "18                                               None   \n",
       "19                                               None   \n",
       "\n",
       "                                   Aggregation method  \\\n",
       "0                                                 Sum   \n",
       "1    horses and mules. FAO estimates that most far...   \n",
       "2                                                None   \n",
       "3                                           CC BY-4.0   \n",
       "4                                    Weighted average   \n",
       "5                                                None   \n",
       "6                                                None   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9                                                None   \n",
       "10                                          CC BY-4.0   \n",
       "11                                   Weighted average   \n",
       "12                                               None   \n",
       "13                                               None   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16                                               None   \n",
       "17                                          CC BY-4.0   \n",
       "18                                                Sum   \n",
       "19                                               None   \n",
       "\n",
       "                           Limitations and exceptions  \\\n",
       "0   The data are collected by the Food and Agricul...   \n",
       "1                             seeds or agrochemicals.   \n",
       "2                                                None   \n",
       "3                                                None   \n",
       "4   The FAO has revised the time series for fertil...   \n",
       "5                                                None   \n",
       "6                                                None   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9                                                None   \n",
       "10                                               None   \n",
       "11  The FAO has revised the time series for fertil...   \n",
       "12                                               None   \n",
       "13                                               None   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16                                               None   \n",
       "17                                               None   \n",
       "18  The data are collected by the Food and Agricul...   \n",
       "19                                               None   \n",
       "\n",
       "   Notes from original source General comments  \\\n",
       "0                        None             None   \n",
       "1                        None             None   \n",
       "2                        None             None   \n",
       "3                        None             None   \n",
       "4                        None             None   \n",
       "5                        None             None   \n",
       "6                        None             None   \n",
       "7                        None             None   \n",
       "8                        None             None   \n",
       "9                        None             None   \n",
       "10                       None             None   \n",
       "11                       None             None   \n",
       "12                       None             None   \n",
       "13                       None             None   \n",
       "14                       None             None   \n",
       "15                       None             None   \n",
       "16                       None             None   \n",
       "17                       None             None   \n",
       "18                       None             None   \n",
       "19                       None             None   \n",
       "\n",
       "                                               Source  \\\n",
       "0   Food and Agriculture Organization, electronic ...   \n",
       "1                                                None   \n",
       "2                                                None   \n",
       "3                                                None   \n",
       "4                                                None   \n",
       "5                                                None   \n",
       "6                                                None   \n",
       "7                                                None   \n",
       "8                                                None   \n",
       "9                                                None   \n",
       "10                                               None   \n",
       "11                                               None   \n",
       "12                                               None   \n",
       "13                                               None   \n",
       "14                                               None   \n",
       "15                                               None   \n",
       "16                                               None   \n",
       "17                                               None   \n",
       "18  Food and Agriculture Organization, electronic ...   \n",
       "19                                               None   \n",
       "\n",
       "                  Statistical concept and methodology Development relevance  \\\n",
       "0   \"A tractor provides the power and traction to ...           for plowing   \n",
       "1                                                None                  None   \n",
       "2                                                None                  None   \n",
       "3                                                None                  None   \n",
       "4                                                None                  None   \n",
       "5                                                None                  None   \n",
       "6                                                None                  None   \n",
       "7                                                None                  None   \n",
       "8                                                None                  None   \n",
       "9                                                None                  None   \n",
       "10                                               None                  None   \n",
       "11                                               None                  None   \n",
       "12                                               None                  None   \n",
       "13                                               None                  None   \n",
       "14                                               None                  None   \n",
       "15                                               None                  None   \n",
       "16                                               None                  None   \n",
       "17                                               None                  None   \n",
       "18  Agricultural land constitutes only a part of a...                  None   \n",
       "19                                               None                  None   \n",
       "\n",
       "   Related source links Other web links Related indicators License Type  \n",
       "0               tilling         disking          harrowing     planting  \n",
       "1                  None            None               None         None  \n",
       "2                  None            None               None         None  \n",
       "3                  None            None               None         None  \n",
       "4                  None            None               None         None  \n",
       "5                  None            None               None         None  \n",
       "6                  None            None               None         None  \n",
       "7                  None            None               None         None  \n",
       "8                  None            None               None         None  \n",
       "9                  None            None               None         None  \n",
       "10                 None            None               None         None  \n",
       "11                 None            None               None         None  \n",
       "12                 None            None               None         None  \n",
       "13                 None            None               None         None  \n",
       "14                 None            None               None         None  \n",
       "15                 None            None               None         None  \n",
       "16                 None            None               None         None  \n",
       "17                 None            None               None         None  \n",
       "18                 None            None               None         None  \n",
       "19                 None            None               None         None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the helper function to display the DataFrame with all columns\n",
    "showDF(df_series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b82fa380",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4284"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get some basic stats\n",
    "df_series.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "684dd6c2-49be-48bb-83a4-1b58f456785b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#Read the file \"WDIData.csv\" into a Spark Data Frame\n",
    "df_indicators = spark.read.format(\"csv\") \\\n",
    "            .option(\"header\", \"true\") \\\n",
    "            .option(\"inferSchema\", \"true\") \\\n",
    "            .load(\"WDIData.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "954ef87c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Country Name: string (nullable = true)\n",
      " |-- Country Code: string (nullable = true)\n",
      " |-- Indicator Name: string (nullable = true)\n",
      " |-- Indicator Code: string (nullable = true)\n",
      " |-- 1960: double (nullable = true)\n",
      " |-- 1961: double (nullable = true)\n",
      " |-- 1962: double (nullable = true)\n",
      " |-- 1963: double (nullable = true)\n",
      " |-- 1964: double (nullable = true)\n",
      " |-- 1965: double (nullable = true)\n",
      " |-- 1966: double (nullable = true)\n",
      " |-- 1967: double (nullable = true)\n",
      " |-- 1968: double (nullable = true)\n",
      " |-- 1969: double (nullable = true)\n",
      " |-- 1970: double (nullable = true)\n",
      " |-- 1971: double (nullable = true)\n",
      " |-- 1972: double (nullable = true)\n",
      " |-- 1973: double (nullable = true)\n",
      " |-- 1974: double (nullable = true)\n",
      " |-- 1975: double (nullable = true)\n",
      " |-- 1976: double (nullable = true)\n",
      " |-- 1977: double (nullable = true)\n",
      " |-- 1978: double (nullable = true)\n",
      " |-- 1979: double (nullable = true)\n",
      " |-- 1980: double (nullable = true)\n",
      " |-- 1981: double (nullable = true)\n",
      " |-- 1982: double (nullable = true)\n",
      " |-- 1983: double (nullable = true)\n",
      " |-- 1984: double (nullable = true)\n",
      " |-- 1985: double (nullable = true)\n",
      " |-- 1986: double (nullable = true)\n",
      " |-- 1987: double (nullable = true)\n",
      " |-- 1988: double (nullable = true)\n",
      " |-- 1989: double (nullable = true)\n",
      " |-- 1990: double (nullable = true)\n",
      " |-- 1991: double (nullable = true)\n",
      " |-- 1992: double (nullable = true)\n",
      " |-- 1993: double (nullable = true)\n",
      " |-- 1994: double (nullable = true)\n",
      " |-- 1995: double (nullable = true)\n",
      " |-- 1996: double (nullable = true)\n",
      " |-- 1997: double (nullable = true)\n",
      " |-- 1998: double (nullable = true)\n",
      " |-- 1999: double (nullable = true)\n",
      " |-- 2000: double (nullable = true)\n",
      " |-- 2001: double (nullable = true)\n",
      " |-- 2002: double (nullable = true)\n",
      " |-- 2003: double (nullable = true)\n",
      " |-- 2004: double (nullable = true)\n",
      " |-- 2005: double (nullable = true)\n",
      " |-- 2006: double (nullable = true)\n",
      " |-- 2007: double (nullable = true)\n",
      " |-- 2008: double (nullable = true)\n",
      " |-- 2009: double (nullable = true)\n",
      " |-- 2010: double (nullable = true)\n",
      " |-- 2011: double (nullable = true)\n",
      " |-- 2012: double (nullable = true)\n",
      " |-- 2013: double (nullable = true)\n",
      " |-- 2014: double (nullable = true)\n",
      " |-- 2015: double (nullable = true)\n",
      " |-- 2016: double (nullable = true)\n",
      " |-- 2017: double (nullable = true)\n",
      " |-- 2018: double (nullable = true)\n",
      " |-- 2019: double (nullable = true)\n",
      " |-- 2020: double (nullable = true)\n",
      " |-- 2021: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Inspect the schema\n",
    "df_indicators.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0eb0cd4f-dfd7-4fc8-a6db-cedab22a4765",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country Name</th>\n",
       "      <th>Country Code</th>\n",
       "      <th>Indicator Name</th>\n",
       "      <th>Indicator Code</th>\n",
       "      <th>1960</th>\n",
       "      <th>1961</th>\n",
       "      <th>1962</th>\n",
       "      <th>1963</th>\n",
       "      <th>1964</th>\n",
       "      <th>1965</th>\n",
       "      <th>...</th>\n",
       "      <th>2012</th>\n",
       "      <th>2013</th>\n",
       "      <th>2014</th>\n",
       "      <th>2015</th>\n",
       "      <th>2016</th>\n",
       "      <th>2017</th>\n",
       "      <th>2018</th>\n",
       "      <th>2019</th>\n",
       "      <th>2020</th>\n",
       "      <th>2021</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Access to clean fuels and technologies for coo...</td>\n",
       "      <td>EG.CFT.ACCS.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>16.559819</td>\n",
       "      <td>16.936004</td>\n",
       "      <td>17.337896</td>\n",
       "      <td>17.687093</td>\n",
       "      <td>18.140971</td>\n",
       "      <td>18.491344</td>\n",
       "      <td>18.825520</td>\n",
       "      <td>19.272212</td>\n",
       "      <td>19.628009</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Access to clean fuels and technologies for coo...</td>\n",
       "      <td>EG.CFT.ACCS.RU.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6.281667</td>\n",
       "      <td>6.499471</td>\n",
       "      <td>6.680066</td>\n",
       "      <td>6.859110</td>\n",
       "      <td>7.016238</td>\n",
       "      <td>7.180364</td>\n",
       "      <td>7.322294</td>\n",
       "      <td>7.517191</td>\n",
       "      <td>7.651598</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Access to clean fuels and technologies for coo...</td>\n",
       "      <td>EG.CFT.ACCS.UR.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>37.601816</td>\n",
       "      <td>37.855399</td>\n",
       "      <td>38.046781</td>\n",
       "      <td>38.326255</td>\n",
       "      <td>38.468426</td>\n",
       "      <td>38.670044</td>\n",
       "      <td>38.722783</td>\n",
       "      <td>38.927016</td>\n",
       "      <td>39.042839</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Access to electricity (% of population)</td>\n",
       "      <td>EG.ELC.ACCS.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>31.844384</td>\n",
       "      <td>31.794160</td>\n",
       "      <td>32.001027</td>\n",
       "      <td>33.871910</td>\n",
       "      <td>38.880173</td>\n",
       "      <td>40.261358</td>\n",
       "      <td>43.061877</td>\n",
       "      <td>44.270860</td>\n",
       "      <td>45.803485</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Access to electricity, rural (% of rural popul...</td>\n",
       "      <td>EG.ELC.ACCS.RU.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>19.402592</td>\n",
       "      <td>18.663502</td>\n",
       "      <td>17.633986</td>\n",
       "      <td>16.464681</td>\n",
       "      <td>24.531436</td>\n",
       "      <td>25.345111</td>\n",
       "      <td>27.449908</td>\n",
       "      <td>29.641760</td>\n",
       "      <td>30.404935</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Access to electricity, urban (% of urban popul...</td>\n",
       "      <td>EG.ELC.ACCS.UR.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>66.945277</td>\n",
       "      <td>67.112206</td>\n",
       "      <td>66.283426</td>\n",
       "      <td>67.080235</td>\n",
       "      <td>69.132292</td>\n",
       "      <td>70.928567</td>\n",
       "      <td>71.866136</td>\n",
       "      <td>73.332842</td>\n",
       "      <td>73.942949</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.FE.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.MA.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.OL.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.40.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.PL.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.60.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.SO.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Account ownership at a financial institution o...</td>\n",
       "      <td>FX.OWN.TOTL.YG.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Adequacy of social insurance programs (% of to...</td>\n",
       "      <td>per_si_allsi.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Adequacy of social protection and labor progra...</td>\n",
       "      <td>per_allsp.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Adequacy of social safety net programs (% of t...</td>\n",
       "      <td>per_sa_allsa.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Adequacy of unemployment benefits and ALMP (% ...</td>\n",
       "      <td>per_lm_alllm.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Adjusted net enrollment rate, primary (% of pr...</td>\n",
       "      <td>SE.PRM.TENR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Country Name Country Code  \\\n",
       "0   Africa Eastern and Southern          AFE   \n",
       "1   Africa Eastern and Southern          AFE   \n",
       "2   Africa Eastern and Southern          AFE   \n",
       "3   Africa Eastern and Southern          AFE   \n",
       "4   Africa Eastern and Southern          AFE   \n",
       "5   Africa Eastern and Southern          AFE   \n",
       "6   Africa Eastern and Southern          AFE   \n",
       "7   Africa Eastern and Southern          AFE   \n",
       "8   Africa Eastern and Southern          AFE   \n",
       "9   Africa Eastern and Southern          AFE   \n",
       "10  Africa Eastern and Southern          AFE   \n",
       "11  Africa Eastern and Southern          AFE   \n",
       "12  Africa Eastern and Southern          AFE   \n",
       "13  Africa Eastern and Southern          AFE   \n",
       "14  Africa Eastern and Southern          AFE   \n",
       "15  Africa Eastern and Southern          AFE   \n",
       "16  Africa Eastern and Southern          AFE   \n",
       "17  Africa Eastern and Southern          AFE   \n",
       "18  Africa Eastern and Southern          AFE   \n",
       "19  Africa Eastern and Southern          AFE   \n",
       "\n",
       "                                       Indicator Name  \\\n",
       "0   Access to clean fuels and technologies for coo...   \n",
       "1   Access to clean fuels and technologies for coo...   \n",
       "2   Access to clean fuels and technologies for coo...   \n",
       "3             Access to electricity (% of population)   \n",
       "4   Access to electricity, rural (% of rural popul...   \n",
       "5   Access to electricity, urban (% of urban popul...   \n",
       "6   Account ownership at a financial institution o...   \n",
       "7   Account ownership at a financial institution o...   \n",
       "8   Account ownership at a financial institution o...   \n",
       "9   Account ownership at a financial institution o...   \n",
       "10  Account ownership at a financial institution o...   \n",
       "11  Account ownership at a financial institution o...   \n",
       "12  Account ownership at a financial institution o...   \n",
       "13  Account ownership at a financial institution o...   \n",
       "14  Account ownership at a financial institution o...   \n",
       "15  Adequacy of social insurance programs (% of to...   \n",
       "16  Adequacy of social protection and labor progra...   \n",
       "17  Adequacy of social safety net programs (% of t...   \n",
       "18  Adequacy of unemployment benefits and ALMP (% ...   \n",
       "19  Adjusted net enrollment rate, primary (% of pr...   \n",
       "\n",
       "              Indicator Code  1960  1961  1962  1963  1964  1965  ...  \\\n",
       "0             EG.CFT.ACCS.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "1          EG.CFT.ACCS.RU.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "2          EG.CFT.ACCS.UR.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "3             EG.ELC.ACCS.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "4          EG.ELC.ACCS.RU.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "5          EG.ELC.ACCS.UR.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "6             FX.OWN.TOTL.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "7          FX.OWN.TOTL.FE.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "8          FX.OWN.TOTL.MA.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "9          FX.OWN.TOTL.OL.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "10         FX.OWN.TOTL.40.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "11         FX.OWN.TOTL.PL.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "12         FX.OWN.TOTL.60.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "13         FX.OWN.TOTL.SO.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "14         FX.OWN.TOTL.YG.ZS   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "15  per_si_allsi.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "16     per_allsp.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "17  per_sa_allsa.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "18  per_lm_alllm.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "19               SE.PRM.TENR   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "         2012       2013       2014       2015       2016       2017  \\\n",
       "0   16.559819  16.936004  17.337896  17.687093  18.140971  18.491344   \n",
       "1    6.281667   6.499471   6.680066   6.859110   7.016238   7.180364   \n",
       "2   37.601816  37.855399  38.046781  38.326255  38.468426  38.670044   \n",
       "3   31.844384  31.794160  32.001027  33.871910  38.880173  40.261358   \n",
       "4   19.402592  18.663502  17.633986  16.464681  24.531436  25.345111   \n",
       "5   66.945277  67.112206  66.283426  67.080235  69.132292  70.928567   \n",
       "6         NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "7         NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "8         NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "9         NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "10        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "11        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "12        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "13        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "14        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "15        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "16        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "17        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "18        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "19        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "\n",
       "         2018       2019       2020  2021  \n",
       "0   18.825520  19.272212  19.628009   NaN  \n",
       "1    7.322294   7.517191   7.651598   NaN  \n",
       "2   38.722783  38.927016  39.042839   NaN  \n",
       "3   43.061877  44.270860  45.803485   NaN  \n",
       "4   27.449908  29.641760  30.404935   NaN  \n",
       "5   71.866136  73.332842  73.942949   NaN  \n",
       "6         NaN        NaN        NaN   NaN  \n",
       "7         NaN        NaN        NaN   NaN  \n",
       "8         NaN        NaN        NaN   NaN  \n",
       "9         NaN        NaN        NaN   NaN  \n",
       "10        NaN        NaN        NaN   NaN  \n",
       "11        NaN        NaN        NaN   NaN  \n",
       "12        NaN        NaN        NaN   NaN  \n",
       "13        NaN        NaN        NaN   NaN  \n",
       "14        NaN        NaN        NaN   NaN  \n",
       "15        NaN        NaN        NaN   NaN  \n",
       "16        NaN        NaN        NaN   NaN  \n",
       "17        NaN        NaN        NaN   NaN  \n",
       "18        NaN        NaN        NaN   NaN  \n",
       "19        NaN        NaN        NaN   NaN  \n",
       "\n",
       "[20 rows x 66 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the helper function to display the DataFrame with all columns\n",
    "showDF(df_indicators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cbd19974",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "383572"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get some basic stats\n",
    "df_indicators.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e94e909e",
   "metadata": {},
   "source": [
    "\n",
    "Data Transformation, Modeling and Quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7dcfb028-6377-4ad9-9631-8cba2d23e210",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>name</th>\n",
       "      <th>long_name</th>\n",
       "      <th>region</th>\n",
       "      <th>income_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AW</td>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AF</td>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Islamic State of Afghanistan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>South Asia</td>\n",
       "      <td>In addition</td>\n",
       "      <td>the World Bank systematically assesses the ap...</td>\n",
       "      <td>this applies to the period 1960-2006. Alterna...</td>\n",
       "      <td>2016</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AO</td>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>People's Republic of Angola</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AL</td>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>Republic of Albania</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AD</td>\n",
       "      <td>AND</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>Principality of Andorra</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AE</td>\n",
       "      <td>ARE</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>Middle East &amp; North Africa</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AR</td>\n",
       "      <td>ARG</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>Argentine Republic</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AM</td>\n",
       "      <td>ARM</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>Republic of Armenia</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AS</td>\n",
       "      <td>ASM</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AG</td>\n",
       "      <td>ATG</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AU</td>\n",
       "      <td>AUS</td>\n",
       "      <td>Australia</td>\n",
       "      <td>Commonwealth of Australia</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AT</td>\n",
       "      <td>AUT</td>\n",
       "      <td>Austria</td>\n",
       "      <td>Republic of Austria</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AZ</td>\n",
       "      <td>AZE</td>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Republic of Azerbaijan</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BI</td>\n",
       "      <td>BDI</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>Republic of Burundi</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Low income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>BE</td>\n",
       "      <td>BEL</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>Kingdom of Belgium</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>BJ</td>\n",
       "      <td>BEN</td>\n",
       "      <td>Benin</td>\n",
       "      <td>Republic of Benin</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_iso_code wb_country_code  \\\n",
       "0                AW             ABW   \n",
       "1                ZH             AFE   \n",
       "2                AF             AFG   \n",
       "3        South Asia     In addition   \n",
       "4                ZI             AFW   \n",
       "5                AO             AGO   \n",
       "6                AL             ALB   \n",
       "7                AD             AND   \n",
       "8                1A             ARB   \n",
       "9                AE             ARE   \n",
       "10               AR             ARG   \n",
       "11               AM             ARM   \n",
       "12               AS             ASM   \n",
       "13               AG             ATG   \n",
       "14               AU             AUS   \n",
       "15               AT             AUT   \n",
       "16               AZ             AZE   \n",
       "17               BI             BDI   \n",
       "18               BE             BEL   \n",
       "19               BJ             BEN   \n",
       "\n",
       "                                                 name  \\\n",
       "0                                               Aruba   \n",
       "1                         Africa Eastern and Southern   \n",
       "2                                         Afghanistan   \n",
       "3    the World Bank systematically assesses the ap...   \n",
       "4                          Africa Western and Central   \n",
       "5                                              Angola   \n",
       "6                                             Albania   \n",
       "7                                             Andorra   \n",
       "8                                          Arab World   \n",
       "9                                United Arab Emirates   \n",
       "10                                          Argentina   \n",
       "11                                            Armenia   \n",
       "12                                     American Samoa   \n",
       "13                                Antigua and Barbuda   \n",
       "14                                          Australia   \n",
       "15                                            Austria   \n",
       "16                                         Azerbaijan   \n",
       "17                                            Burundi   \n",
       "18                                            Belgium   \n",
       "19                                              Benin   \n",
       "\n",
       "                                            long_name  \\\n",
       "0                                               Aruba   \n",
       "1                         Africa Eastern and Southern   \n",
       "2                        Islamic State of Afghanistan   \n",
       "3    this applies to the period 1960-2006. Alterna...   \n",
       "4                          Africa Western and Central   \n",
       "5                         People's Republic of Angola   \n",
       "6                                 Republic of Albania   \n",
       "7                             Principality of Andorra   \n",
       "8                                          Arab World   \n",
       "9                                United Arab Emirates   \n",
       "10                                 Argentine Republic   \n",
       "11                                Republic of Armenia   \n",
       "12                                     American Samoa   \n",
       "13                                Antigua and Barbuda   \n",
       "14                          Commonwealth of Australia   \n",
       "15                                Republic of Austria   \n",
       "16                             Republic of Azerbaijan   \n",
       "17                                Republic of Burundi   \n",
       "18                                 Kingdom of Belgium   \n",
       "19                                  Republic of Benin   \n",
       "\n",
       "                        region         income_group  \n",
       "0    Latin America & Caribbean          High income  \n",
       "1                         None                 None  \n",
       "2                         None                 None  \n",
       "3                         2016                 None  \n",
       "4                         None                 None  \n",
       "5           Sub-Saharan Africa  Lower middle income  \n",
       "6        Europe & Central Asia  Upper middle income  \n",
       "7        Europe & Central Asia          High income  \n",
       "8                         None                 None  \n",
       "9   Middle East & North Africa          High income  \n",
       "10   Latin America & Caribbean  Upper middle income  \n",
       "11       Europe & Central Asia  Upper middle income  \n",
       "12         East Asia & Pacific  Upper middle income  \n",
       "13   Latin America & Caribbean          High income  \n",
       "14         East Asia & Pacific          High income  \n",
       "15       Europe & Central Asia          High income  \n",
       "16       Europe & Central Asia  Upper middle income  \n",
       "17          Sub-Saharan Africa           Low income  \n",
       "18       Europe & Central Asia          High income  \n",
       "19          Sub-Saharan Africa  Lower middle income  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Select and rename columns needed for our data model for data ingestion.\n",
    "countryDim = df_country \\\n",
    "             .select(\"2-alpha code\", \"Country Code\", \"Short Name\", \"Long Name\", \"Region\", \"Income Group\") \\\n",
    "             .withColumnRenamed(\"2-alpha code\", \"country_iso_code\") \\\n",
    "             .withColumnRenamed(\"Country Code\", \"wb_country_code\") \\\n",
    "             .withColumnRenamed(\"Short Name\", \"name\") \\\n",
    "             .withColumnRenamed(\"Long Name\", \"long_name\") \\\n",
    "             .withColumnRenamed(\"Region\", \"region\") \\\n",
    "             .withColumnRenamed(\"Income Group\", \"income_group\")\n",
    "showDF(countryDim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d6a7f6d1-565c-4dec-ba3d-8d0cb1472d13",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter the DataFrame to include only rows where the country_iso_code is not null\n",
    "countryDimNotNull = countryDim.filter(\"country_iso_code is not null\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b19402",
   "metadata": {},
   "source": [
    "Data Quality Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "80fb6c4b-521d-4e68-8257-2be5f9d2de8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_length</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_length  cnt\n",
       "0             19    4\n",
       "1              2  265"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Do all countries have 2 character country_iso_codes ?\n",
    "showDF( \\\n",
    "    countryDimNotNull \\\n",
    "       .select(F.col(\"country_iso_code\"), \\\n",
    "               F.length(F.col(\"country_iso_code\")).alias(\"column_length\")) \\\n",
    "       .groupBy(\"column_length\") \\\n",
    "       .agg(F.count(\"*\").alias(\"cnt\")) \\\n",
    "       .filter(\"cnt > 1\") \\\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "799227cd-2b40-4ec2-9e3a-bf07f26176e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "265\n"
     ]
    }
   ],
   "source": [
    "# Filter the DataFrame to keep only rows where country_iso_code length is 2\n",
    "fixIsoCountryCode = countryDimNotNull.filter(\"length(country_iso_code) = 2\")\n",
    "countAfterFiltering = fixIsoCountryCode.count()\n",
    "print(countAfterFiltering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2201db19-dee1-46f4-b9a3-73713eb5fa0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_length</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_length  cnt\n",
       "0              3  264\n",
       "1             20    1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Check the length distribution of wb_country_code after filtering\n",
    "WbCountryCodeLength = (fixIsoCountryCode \\\n",
    "    .select(\"wb_country_code\",\n",
    "            F.length(\"wb_country_code\").alias(\"column_length\"))\\\n",
    "    .groupBy(\"column_length\") \\\n",
    "    .agg(F.count(\"*\").alias(\"cnt\")) \n",
    "                      )\n",
    "showDF(WbCountryCodeLength)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3d89b0d4-af9b-42e1-8f36-7a07fefc1f91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>name</th>\n",
       "      <th>long_name</th>\n",
       "      <th>region</th>\n",
       "      <th>income_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>F1</td>\n",
       "      <td>For more information</td>\n",
       "      <td>please visit https://www.worldbank.org/en/top...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country_iso_code       wb_country_code  \\\n",
       "0               F1  For more information   \n",
       "\n",
       "                                                name long_name region  \\\n",
       "0   please visit https://www.worldbank.org/en/top...      None   None   \n",
       "\n",
       "  income_group  \n",
       "0         None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Display rows where `wb_country_code` has a length of 20 to assess if these entries should be retained or removed\n",
    "showDF(fixIsoCountryCode.filter(\"length(wb_country_code) = 20\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "174e8a21-2a23-4c1d-9905-f5ebca8ecbcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "264\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>name</th>\n",
       "      <th>long_name</th>\n",
       "      <th>region</th>\n",
       "      <th>income_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AW</td>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AF</td>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Islamic State of Afghanistan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AO</td>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>People's Republic of Angola</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AL</td>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>Republic of Albania</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AD</td>\n",
       "      <td>AND</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>Principality of Andorra</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AE</td>\n",
       "      <td>ARE</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>Middle East &amp; North Africa</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AR</td>\n",
       "      <td>ARG</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>Argentine Republic</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AM</td>\n",
       "      <td>ARM</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>Republic of Armenia</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AS</td>\n",
       "      <td>ASM</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AG</td>\n",
       "      <td>ATG</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AU</td>\n",
       "      <td>AUS</td>\n",
       "      <td>Australia</td>\n",
       "      <td>Commonwealth of Australia</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AT</td>\n",
       "      <td>AUT</td>\n",
       "      <td>Austria</td>\n",
       "      <td>Republic of Austria</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AZ</td>\n",
       "      <td>AZE</td>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Republic of Azerbaijan</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>BI</td>\n",
       "      <td>BDI</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>Republic of Burundi</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Low income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BE</td>\n",
       "      <td>BEL</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>Kingdom of Belgium</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>BJ</td>\n",
       "      <td>BEN</td>\n",
       "      <td>Benin</td>\n",
       "      <td>Republic of Benin</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>BF</td>\n",
       "      <td>BFA</td>\n",
       "      <td>Burkina Faso</td>\n",
       "      <td>Burkina Faso</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Low income</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_iso_code wb_country_code                         name  \\\n",
       "0                AW             ABW                        Aruba   \n",
       "1                ZH             AFE  Africa Eastern and Southern   \n",
       "2                AF             AFG                  Afghanistan   \n",
       "3                ZI             AFW   Africa Western and Central   \n",
       "4                AO             AGO                       Angola   \n",
       "5                AL             ALB                      Albania   \n",
       "6                AD             AND                      Andorra   \n",
       "7                1A             ARB                   Arab World   \n",
       "8                AE             ARE         United Arab Emirates   \n",
       "9                AR             ARG                    Argentina   \n",
       "10               AM             ARM                      Armenia   \n",
       "11               AS             ASM               American Samoa   \n",
       "12               AG             ATG          Antigua and Barbuda   \n",
       "13               AU             AUS                    Australia   \n",
       "14               AT             AUT                      Austria   \n",
       "15               AZ             AZE                   Azerbaijan   \n",
       "16               BI             BDI                      Burundi   \n",
       "17               BE             BEL                      Belgium   \n",
       "18               BJ             BEN                        Benin   \n",
       "19               BF             BFA                 Burkina Faso   \n",
       "\n",
       "                       long_name                      region  \\\n",
       "0                          Aruba   Latin America & Caribbean   \n",
       "1    Africa Eastern and Southern                        None   \n",
       "2   Islamic State of Afghanistan                        None   \n",
       "3     Africa Western and Central                        None   \n",
       "4    People's Republic of Angola          Sub-Saharan Africa   \n",
       "5            Republic of Albania       Europe & Central Asia   \n",
       "6        Principality of Andorra       Europe & Central Asia   \n",
       "7                     Arab World                        None   \n",
       "8           United Arab Emirates  Middle East & North Africa   \n",
       "9             Argentine Republic   Latin America & Caribbean   \n",
       "10           Republic of Armenia       Europe & Central Asia   \n",
       "11                American Samoa         East Asia & Pacific   \n",
       "12           Antigua and Barbuda   Latin America & Caribbean   \n",
       "13     Commonwealth of Australia         East Asia & Pacific   \n",
       "14           Republic of Austria       Europe & Central Asia   \n",
       "15        Republic of Azerbaijan       Europe & Central Asia   \n",
       "16           Republic of Burundi          Sub-Saharan Africa   \n",
       "17            Kingdom of Belgium       Europe & Central Asia   \n",
       "18             Republic of Benin          Sub-Saharan Africa   \n",
       "19                  Burkina Faso          Sub-Saharan Africa   \n",
       "\n",
       "           income_group  \n",
       "0           High income  \n",
       "1                  None  \n",
       "2                  None  \n",
       "3                  None  \n",
       "4   Lower middle income  \n",
       "5   Upper middle income  \n",
       "6           High income  \n",
       "7                  None  \n",
       "8           High income  \n",
       "9   Upper middle income  \n",
       "10  Upper middle income  \n",
       "11  Upper middle income  \n",
       "12          High income  \n",
       "13          High income  \n",
       "14          High income  \n",
       "15  Upper middle income  \n",
       "16           Low income  \n",
       "17          High income  \n",
       "18  Lower middle income  \n",
       "19           Low income  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Filter the DataFrame to keep only rows where wb_country_code length is 3\n",
    "fixWbCountryCode = fixIsoCountryCode.filter(\"length(wb_country_code) = 3\")\n",
    "print(fixWbCountryCode .count())\n",
    "showDF(fixWbCountryCode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fe241d4a-5ccd-4bf0-89a8-ed246ab2a25f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [country_iso_code, cnt]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [wb_country_code, cnt]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [name, cnt]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Check duplicate records for the columns\n",
    "duplicateISOCountryCode = fixWbCountryCode.groupby(\"country_iso_code\").agg(F.count(\"*\").alias(\"cnt\")).filter(\"cnt>1\")\n",
    "showDF(duplicateISOCountryCode)               \n",
    "duplicateWBCountryCode = fixWbCountryCode.groupby(\"wb_country_code\").agg(F.count(\"*\").alias(\"cnt\")).filter(\"cnt>1\")\n",
    "showDF(duplicateWBCountryCode)                \n",
    "duplicateCountryName = fixWbCountryCode.groupby(\"name\").agg(F.count(\"*\").alias(\"cnt\")).filter(\"cnt>1\")\n",
    "showDF(duplicateCountryName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0fa81a8b-d9ee-4acc-ac18-ac5bbd3365ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "264\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>name</th>\n",
       "      <th>long_name</th>\n",
       "      <th>region</th>\n",
       "      <th>income_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AW</td>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AF</td>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Islamic State of Afghanistan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AO</td>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>People's Republic of Angola</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AL</td>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>Republic of Albania</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AD</td>\n",
       "      <td>AND</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>Principality of Andorra</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AE</td>\n",
       "      <td>ARE</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>Middle East &amp; North Africa</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AR</td>\n",
       "      <td>ARG</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>Argentine Republic</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AM</td>\n",
       "      <td>ARM</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>Republic of Armenia</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AS</td>\n",
       "      <td>ASM</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>American Samoa</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AG</td>\n",
       "      <td>ATG</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>Latin America &amp; Caribbean</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AU</td>\n",
       "      <td>AUS</td>\n",
       "      <td>Australia</td>\n",
       "      <td>Commonwealth of Australia</td>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AT</td>\n",
       "      <td>AUT</td>\n",
       "      <td>Austria</td>\n",
       "      <td>Republic of Austria</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AZ</td>\n",
       "      <td>AZE</td>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Republic of Azerbaijan</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>Upper middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>BI</td>\n",
       "      <td>BDI</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>Republic of Burundi</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Low income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BE</td>\n",
       "      <td>BEL</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>Kingdom of Belgium</td>\n",
       "      <td>Europe &amp; Central Asia</td>\n",
       "      <td>High income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>BJ</td>\n",
       "      <td>BEN</td>\n",
       "      <td>Benin</td>\n",
       "      <td>Republic of Benin</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Lower middle income</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>BF</td>\n",
       "      <td>BFA</td>\n",
       "      <td>Burkina Faso</td>\n",
       "      <td>Burkina Faso</td>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Low income</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_iso_code wb_country_code                         name  \\\n",
       "0                AW             ABW                        Aruba   \n",
       "1                ZH             AFE  Africa Eastern and Southern   \n",
       "2                AF             AFG                  Afghanistan   \n",
       "3                ZI             AFW   Africa Western and Central   \n",
       "4                AO             AGO                       Angola   \n",
       "5                AL             ALB                      Albania   \n",
       "6                AD             AND                      Andorra   \n",
       "7                1A             ARB                   Arab World   \n",
       "8                AE             ARE         United Arab Emirates   \n",
       "9                AR             ARG                    Argentina   \n",
       "10               AM             ARM                      Armenia   \n",
       "11               AS             ASM               American Samoa   \n",
       "12               AG             ATG          Antigua and Barbuda   \n",
       "13               AU             AUS                    Australia   \n",
       "14               AT             AUT                      Austria   \n",
       "15               AZ             AZE                   Azerbaijan   \n",
       "16               BI             BDI                      Burundi   \n",
       "17               BE             BEL                      Belgium   \n",
       "18               BJ             BEN                        Benin   \n",
       "19               BF             BFA                 Burkina Faso   \n",
       "\n",
       "                       long_name                      region  \\\n",
       "0                          Aruba   Latin America & Caribbean   \n",
       "1    Africa Eastern and Southern                        None   \n",
       "2   Islamic State of Afghanistan                        None   \n",
       "3     Africa Western and Central                        None   \n",
       "4    People's Republic of Angola          Sub-Saharan Africa   \n",
       "5            Republic of Albania       Europe & Central Asia   \n",
       "6        Principality of Andorra       Europe & Central Asia   \n",
       "7                     Arab World                        None   \n",
       "8           United Arab Emirates  Middle East & North Africa   \n",
       "9             Argentine Republic   Latin America & Caribbean   \n",
       "10           Republic of Armenia       Europe & Central Asia   \n",
       "11                American Samoa         East Asia & Pacific   \n",
       "12           Antigua and Barbuda   Latin America & Caribbean   \n",
       "13     Commonwealth of Australia         East Asia & Pacific   \n",
       "14           Republic of Austria       Europe & Central Asia   \n",
       "15        Republic of Azerbaijan       Europe & Central Asia   \n",
       "16           Republic of Burundi          Sub-Saharan Africa   \n",
       "17            Kingdom of Belgium       Europe & Central Asia   \n",
       "18             Republic of Benin          Sub-Saharan Africa   \n",
       "19                  Burkina Faso          Sub-Saharan Africa   \n",
       "\n",
       "           income_group  \n",
       "0           High income  \n",
       "1                  None  \n",
       "2                  None  \n",
       "3                  None  \n",
       "4   Lower middle income  \n",
       "5   Upper middle income  \n",
       "6           High income  \n",
       "7                  None  \n",
       "8           High income  \n",
       "9   Upper middle income  \n",
       "10  Upper middle income  \n",
       "11  Upper middle income  \n",
       "12          High income  \n",
       "13          High income  \n",
       "14          High income  \n",
       "15  Upper middle income  \n",
       "16           Low income  \n",
       "17          High income  \n",
       "18  Lower middle income  \n",
       "19           Low income  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "countryDimFinal = fixWbCountryCode\n",
    "print(countryDimFinal.count())\n",
    "showDF(countryDimFinal)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40ba3dfb",
   "metadata": {},
   "source": [
    "Write the data to the destination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ae146457",
   "metadata": {},
   "outputs": [],
   "source": [
    "# write the country dimension to an output csv file\n",
    "countryDimFinal \\\n",
    "    .coalesce(1) \\\n",
    "    .write.csv('output/CountryDim', mode='overwrite', header='true')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b7099fae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "country_iso_code,wb_country_code,name,long_name,region,income_group\n",
      "AW,ABW,Aruba,Aruba,Latin America & Caribbean,High income\n",
      "ZH,AFE,Africa Eastern and Southern,Africa Eastern and Southern,,\n",
      "AF,AFG,Afghanistan,Islamic State of Afghanistan,,\n",
      "ZI,AFW,Africa Western and Central,Africa Western and Central,,\n",
      "AO,AGO,Angola,People's Republic of Angola,Sub-Saharan Africa,Lower middle income\n",
      "AL,ALB,Albania,Republic of Albania,Europe & Central Asia,Upper middle income\n",
      "AD,AND,Andorra,Principality of Andorra,Europe & Central Asia,High income\n",
      "1A,ARB,Arab World,Arab World,,\n",
      "AE,ARE,United Arab Emirates,United Arab Emirates,Middle East & North Africa,High income\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cat output/CountryDim/*csv | head"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56b98ca1",
   "metadata": {},
   "source": [
    "Transform the Series Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "bd87d4ef-72f0-43e8-a942-71250fd9683e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_name</th>\n",
       "      <th>periodicity</th>\n",
       "      <th>Aggregation_method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AG.AGR.TRAC.NO</td>\n",
       "      <td>Agricultural machinery, tractors</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AG.CON.FERT.PT.ZS</td>\n",
       "      <td>Fertilizer consumption (% of fertilizer produc...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AG.CON.FERT.ZS</td>\n",
       "      <td>Fertilizer consumption (kilograms per hectare ...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AG.LND.AGRI.K2</td>\n",
       "      <td>Agricultural land (sq. km)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AG.LND.AGRI.ZS</td>\n",
       "      <td>Agricultural land (% of land area)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AG.LND.ARBL.HA</td>\n",
       "      <td>Arable land (hectares)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AG.LND.ARBL.HA.PC</td>\n",
       "      <td>Arable land (hectares per person)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AG.LND.ARBL.ZS</td>\n",
       "      <td>Arable land (% of land area)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AG.LND.CREL.HA</td>\n",
       "      <td>Land under cereal production (hectares)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AG.LND.CROP.ZS</td>\n",
       "      <td>Permanent cropland (% of land area)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AG.LND.EL5M.RU.K2</td>\n",
       "      <td>Rural land area where elevation is below 5 met...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AG.LND.EL5M.RU.ZS</td>\n",
       "      <td>Rural land area where elevation is below 5 met...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AG.LND.EL5M.UR.K2</td>\n",
       "      <td>Urban land area where elevation is below 5 met...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AG.LND.EL5M.UR.ZS</td>\n",
       "      <td>Urban land area where elevation is below 5 met...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area where elevation is below 5 meters (%...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AG.LND.FRST.K2</td>\n",
       "      <td>Forest area (sq. km)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AG.LND.FRST.ZS</td>\n",
       "      <td>Forest area (% of land area)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>AG.LND.IRIG.AG.ZS</td>\n",
       "      <td>Agricultural irrigated land (% of total agricu...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AG.LND.PRCP.MM</td>\n",
       "      <td>Average precipitation in depth (mm per year)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>AG.LND.TOTL.K2</td>\n",
       "      <td>Land area (sq. km)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Sum</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       indicator_code                                     indicator_name  \\\n",
       "0      AG.AGR.TRAC.NO                   Agricultural machinery, tractors   \n",
       "1   AG.CON.FERT.PT.ZS  Fertilizer consumption (% of fertilizer produc...   \n",
       "2      AG.CON.FERT.ZS  Fertilizer consumption (kilograms per hectare ...   \n",
       "3      AG.LND.AGRI.K2                         Agricultural land (sq. km)   \n",
       "4      AG.LND.AGRI.ZS                 Agricultural land (% of land area)   \n",
       "5      AG.LND.ARBL.HA                             Arable land (hectares)   \n",
       "6   AG.LND.ARBL.HA.PC                  Arable land (hectares per person)   \n",
       "7      AG.LND.ARBL.ZS                       Arable land (% of land area)   \n",
       "8      AG.LND.CREL.HA            Land under cereal production (hectares)   \n",
       "9      AG.LND.CROP.ZS                Permanent cropland (% of land area)   \n",
       "10  AG.LND.EL5M.RU.K2  Rural land area where elevation is below 5 met...   \n",
       "11  AG.LND.EL5M.RU.ZS  Rural land area where elevation is below 5 met...   \n",
       "12  AG.LND.EL5M.UR.K2  Urban land area where elevation is below 5 met...   \n",
       "13  AG.LND.EL5M.UR.ZS  Urban land area where elevation is below 5 met...   \n",
       "14     AG.LND.EL5M.ZS  Land area where elevation is below 5 meters (%...   \n",
       "15     AG.LND.FRST.K2                               Forest area (sq. km)   \n",
       "16     AG.LND.FRST.ZS                       Forest area (% of land area)   \n",
       "17  AG.LND.IRIG.AG.ZS  Agricultural irrigated land (% of total agricu...   \n",
       "18     AG.LND.PRCP.MM       Average precipitation in depth (mm per year)   \n",
       "19     AG.LND.TOTL.K2                                 Land area (sq. km)   \n",
       "\n",
       "   periodicity Aggregation_method  \n",
       "0       Annual                Sum  \n",
       "1       Annual   Weighted average  \n",
       "2       Annual   Weighted average  \n",
       "3       Annual                Sum  \n",
       "4       Annual   Weighted average  \n",
       "5       Annual               None  \n",
       "6       Annual   Weighted average  \n",
       "7       Annual   Weighted average  \n",
       "8       Annual                Sum  \n",
       "9       Annual   Weighted average  \n",
       "10      Annual                Sum  \n",
       "11      Annual   Weighted average  \n",
       "12      Annual                Sum  \n",
       "13      Annual   Weighted average  \n",
       "14      Annual   Weighted average  \n",
       "15      Annual                Sum  \n",
       "16      Annual   Weighted average  \n",
       "17      Annual   Weighted average  \n",
       "18      Annual               None  \n",
       "19      Annual                Sum  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1377"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter only for series that have Annual periodicity\n",
    "# Select and rename columns needed for our data model for data ingestion.\n",
    "seriesDim = df_series\\\n",
    "            .select(\"Series Code\", \"Indicator Name\", \"Periodicity\", \"Aggregation method\") \\\n",
    "            .withColumnRenamed(\"Series Code\", \"indicator_code\") \\\n",
    "            .withColumnRenamed(\"Indicator Name\", \"indicator_name\") \\\n",
    "            .withColumnRenamed(\"Periodicity\", \"periodicity\") \\\n",
    "            .withColumnRenamed(\"Aggregation method\", \"Aggregation_method\") \\\n",
    "            .filter(F.col(\"Periodicity\") == \"Annual\")\n",
    "showDF(seriesDim)\n",
    "seriesDim.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9643bc44-82e8-4cae-b5b6-fbc2579abc3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1377"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter rows where `indicator_code` is not null\n",
    "seriesNotNull = seriesDim.filter(\"indicator_code is Not Null\")\n",
    "seriesNotNull.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "170c1a30-290e-437b-b9bd-5e889cbd9ed1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>be caused by household chemicals</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     indicator_code  cnt\n",
       "0  be caused by household chemicals    2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Check for duplicate entries\n",
    "duplicate_check_indicator_code = seriesNotNull.groupBy(\"indicator_code\").agg(F.count(\"*\").alias(\"cnt\")).filter(\"cnt>1\")\n",
    "showDF(duplicate_check_indicator_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dd8521a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1376"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop duplicate entries\n",
    "seriesNoDuplicates = seriesNotNull.dropDuplicates([\"indicator_code\"])\n",
    "seriesNoDuplicates.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b359087c",
   "metadata": {},
   "source": [
    "Cellular and Broadband Penetration Analysis:\n",
    "\n",
    "We aim to measure cellular and broadband penetration in comparison to population demographics for each country. Additionally, we seek insights on annual global aggregates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "f024efba-cce3-4089-9885-c489538ce409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AG.AGR.TRAC.NO</td>\n",
       "      <td>Agricultural machinery, tractors</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AG.LND.AGRI.K2</td>\n",
       "      <td>Agricultural land (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AG.LND.CREL.HA</td>\n",
       "      <td>Land under cereal production (hectares)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AG.LND.EL5M.RU.K2</td>\n",
       "      <td>Rural land area where elevation is below 5 meters (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AG.LND.EL5M.UR.K2</td>\n",
       "      <td>Urban land area where elevation is below 5 meters (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AG.LND.FRST.K2</td>\n",
       "      <td>Forest area (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AG.LND.TOTL.K2</td>\n",
       "      <td>Land area (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AG.LND.TOTL.RU.K2</td>\n",
       "      <td>Rural land area (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AG.LND.TOTL.UR.K2</td>\n",
       "      <td>Urban land area (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AG.PRD.CREL.MT</td>\n",
       "      <td>Cereal production (metric tons)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AG.SRF.TOTL.K2</td>\n",
       "      <td>Surface area (sq. km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>BM.GSR.FCTY.CD</td>\n",
       "      <td>Primary income payments (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>BM.GSR.ROYL.CD</td>\n",
       "      <td>Charges for the use of intellectual property, payments (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>BM.KLT.DINV.CD.WD</td>\n",
       "      <td>Foreign direct investment, net outflows (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>BM.TRF.PRVT.CD</td>\n",
       "      <td>Secondary income, other sectors, payments (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>BM.TRF.PWKR.CD.DT</td>\n",
       "      <td>Personal remittances, paid (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>BX.GRT.EXTA.CD.WD</td>\n",
       "      <td>Grants, excluding technical cooperation (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BX.GRT.TECH.CD.WD</td>\n",
       "      <td>Technical cooperation grants (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>BX.GSR.CCIS.CD</td>\n",
       "      <td>ICT service exports (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>BX.GSR.FCTY.CD</td>\n",
       "      <td>Primary income receipts (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>BX.GSR.ROYL.CD</td>\n",
       "      <td>Charges for the use of intellectual property, receipts (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>BX.KLT.DINV.CD.WD</td>\n",
       "      <td>Foreign direct investment, net inflows (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>BX.PEF.TOTL.CD.WD</td>\n",
       "      <td>Portfolio equity, net inflows (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>BX.TRF.CURR.CD</td>\n",
       "      <td>Secondary income receipts (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>BX.TRF.PWKR.CD</td>\n",
       "      <td>Personal transfers, receipts (BoP, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>BX.TRF.PWKR.CD.DT</td>\n",
       "      <td>Personal remittances, received (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>CM.MKT.LCAP.CD</td>\n",
       "      <td>Market capitalization of listed domestic companies (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>CM.MKT.LDOM.NO</td>\n",
       "      <td>Listed domestic companies, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>CM.MKT.TRAD.CD</td>\n",
       "      <td>Stocks traded, total value (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>DC.DAC.AUSL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Australia (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>DC.DAC.AUTL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Austria (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>DC.DAC.BELL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Belgium (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>DC.DAC.CANL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Canada (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>DC.DAC.CECL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, European Union institutions (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>DC.DAC.CHEL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Switzerland (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>DC.DAC.CZEL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Czech Republic (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>DC.DAC.DEUL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Germany (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>DC.DAC.DNKL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Denmark (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>DC.DAC.ESPL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Spain (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>DC.DAC.FINL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Finland (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>DC.DAC.FRAL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, France (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>DC.DAC.GBRL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, United Kingdom (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>DC.DAC.GRCL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Greece (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>DC.DAC.HUNL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Hungary (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>DC.DAC.IRLL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Ireland (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>DC.DAC.ISLL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Iceland (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>DC.DAC.ITAL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Italy (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>DC.DAC.JPNL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Japan (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>DC.DAC.KORL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Korea, Rep. (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>DC.DAC.LUXL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Luxembourg (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>DC.DAC.NLDL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Netherlands (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>DC.DAC.NORL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Norway (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>DC.DAC.NZLL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, New Zealand (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>DC.DAC.POLL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Poland (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>DC.DAC.PRTL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Portugal (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>DC.DAC.SVKL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Slovak Republic (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>DC.DAC.SVNL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Slovenia (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>DC.DAC.SWEL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Sweden (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>DC.DAC.TOTL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, Total (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>DC.DAC.USAL.CD</td>\n",
       "      <td>Net bilateral aid flows from DAC donors, United States (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>DT.DOD.DECT.CD</td>\n",
       "      <td>External debt stocks, total (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>DT.DOD.DIMF.CD</td>\n",
       "      <td>Use of IMF credit (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>DT.DOD.DLXF.CD</td>\n",
       "      <td>External debt stocks, long-term (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>DT.DOD.DPNG.CD</td>\n",
       "      <td>External debt stocks, private nonguaranteed (PNG) (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>DT.DOD.DPPG.CD</td>\n",
       "      <td>External debt stocks, public and publicly guaranteed (PPG) (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>DT.DOD.DSTC.CD</td>\n",
       "      <td>External debt stocks, short-term (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>DT.DOD.MIBR.CD</td>\n",
       "      <td>PPG, IBRD (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>DT.DOD.MIDA.CD</td>\n",
       "      <td>PPG, IDA (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>DT.DOD.MWBG.CD</td>\n",
       "      <td>IBRD loans and IDA credits (DOD, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>DT.DOD.PVLX.CD</td>\n",
       "      <td>Present value of external debt (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>DT.NFL.BLAT.CD</td>\n",
       "      <td>Net financial flows, bilateral (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>DT.NFL.BOND.CD</td>\n",
       "      <td>Portfolio investment, bonds (PPG + PNG) (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>DT.NFL.DPNG.CD</td>\n",
       "      <td>Net flows on external debt, private nonguaranteed (PNG) (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>DT.NFL.FAOG.CD</td>\n",
       "      <td>Net official flows from UN agencies, FAO (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>DT.NFL.IAEA.CD</td>\n",
       "      <td>Net official flows from UN agencies, IAEA (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>DT.NFL.IFAD.CD</td>\n",
       "      <td>Net official flows from UN agencies, IFAD (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>DT.NFL.ILOG.CD</td>\n",
       "      <td>Net official flows from UN agencies, ILO (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>DT.NFL.IMFC.CD</td>\n",
       "      <td>Net financial flows, IMF concessional (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>DT.NFL.IMFN.CD</td>\n",
       "      <td>Net financial flows, IMF nonconcessional (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>DT.NFL.MIBR.CD</td>\n",
       "      <td>Net financial flows, IBRD (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>DT.NFL.MIDA.CD</td>\n",
       "      <td>Net financial flows, IDA (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>DT.NFL.MLAT.CD</td>\n",
       "      <td>Net financial flows, multilateral (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>DT.NFL.MOTH.CD</td>\n",
       "      <td>Net financial flows, others (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>DT.NFL.NIFC.CD</td>\n",
       "      <td>IFC, private nonguaranteed (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>DT.NFL.OFFT.CD</td>\n",
       "      <td>PPG, official creditors (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>DT.NFL.PBND.CD</td>\n",
       "      <td>PPG, bonds (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>DT.NFL.PCBK.CD</td>\n",
       "      <td>PPG, commercial banks (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>DT.NFL.PCBO.CD</td>\n",
       "      <td>Commercial banks and other lending (PPG + PNG) (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>DT.NFL.PNGB.CD</td>\n",
       "      <td>PNG, bonds (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>DT.NFL.PNGC.CD</td>\n",
       "      <td>PNG, commercial banks and other creditors (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>DT.NFL.PROP.CD</td>\n",
       "      <td>PPG, other private creditors (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>DT.NFL.PRVT.CD</td>\n",
       "      <td>PPG, private creditors (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>DT.NFL.RDBC.CD</td>\n",
       "      <td>Net financial flows, RDB concessional (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>DT.NFL.RDBN.CD</td>\n",
       "      <td>Net financial flows, RDB nonconcessional (NFL, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>DT.NFL.UNAI.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNAIDS (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>DT.NFL.UNCF.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNICEF (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>DT.NFL.UNCR.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNHCR (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>DT.NFL.UNDP.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNDP (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>DT.NFL.UNEC.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNECE (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>DT.NFL.UNEP.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNEP (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>DT.NFL.UNFP.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNFPA (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>DT.NFL.UNID.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNIDIR (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>DT.NFL.UNPB.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNPBF (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>DT.NFL.UNRW.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNRWA (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>DT.NFL.UNTA.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNTA (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>DT.NFL.UNWT.CD</td>\n",
       "      <td>Net official flows from UN agencies, UNWTO (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>DT.NFL.WFPG.CD</td>\n",
       "      <td>Net official flows from UN agencies, WFP (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>DT.NFL.WHOL.CD</td>\n",
       "      <td>Net official flows from UN agencies, WHO (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>DT.ODA.ALLD.CD</td>\n",
       "      <td>Net official development assistance and official aid received (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>DT.ODA.ALLD.KD</td>\n",
       "      <td>Net official development assistance and official aid received (constant 2020 US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>DT.ODA.OATL.CD</td>\n",
       "      <td>Net official aid received (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>DT.ODA.OATL.KD</td>\n",
       "      <td>Net official aid received (constant 2020 US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>DT.ODA.ODAT.CD</td>\n",
       "      <td>Net official development assistance received (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>DT.ODA.ODAT.KD</td>\n",
       "      <td>Net official development assistance received (constant 2020 US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>DT.TDS.DECT.CD</td>\n",
       "      <td>Debt service on external debt, total (TDS, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>DT.TDS.DIMF.CD</td>\n",
       "      <td>IMF repurchases and charges (TDS, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>DT.TDS.DPPG.CD</td>\n",
       "      <td>Debt service on external debt, public and publicly guaranteed (PPG) (TDS, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>DT.TDS.MLAT.CD</td>\n",
       "      <td>Multilateral debt service (TDS, current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>EG.ELC.RNWX.KH</td>\n",
       "      <td>Electricity production from renewable sources, excluding hydroelectric (kWh)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>EN.ATM.GHGO.KT.CE</td>\n",
       "      <td>Other greenhouse gas emissions, HFC, PFC and SF6 (thousand metric tons of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>EN.ATM.GHGT.KT.CE</td>\n",
       "      <td>Total greenhouse gas emissions (kt of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>EN.ATM.METH.AG.KT.CE</td>\n",
       "      <td>Agricultural methane emissions (thousand metric tons of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>EN.ATM.METH.EG.KT.CE</td>\n",
       "      <td>Methane emissions in energy sector (thousand metric tons of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>EN.ATM.METH.KT.CE</td>\n",
       "      <td>Methane emissions (kt of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>EN.ATM.NOXE.AG.KT.CE</td>\n",
       "      <td>Agricultural nitrous oxide emissions (thousand metric tons of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>EN.ATM.NOXE.EG.KT.CE</td>\n",
       "      <td>Nitrous oxide emissions in energy sector (thousand metric tons of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>EN.ATM.NOXE.KT.CE</td>\n",
       "      <td>Nitrous oxide emissions (thousand metric tons of CO2 equivalent)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>EN.BIR.THRD.NO</td>\n",
       "      <td>Bird species, threatened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>EN.FSH.THRD.NO</td>\n",
       "      <td>Fish species, threatened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>EN.HPT.THRD.NO</td>\n",
       "      <td>Plant species (higher), threatened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>EN.MAM.THRD.NO</td>\n",
       "      <td>Mammal species, threatened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>ER.FSH.AQUA.MT</td>\n",
       "      <td>Aquaculture production (metric tons)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>ER.FSH.CAPT.MT</td>\n",
       "      <td>Capture fisheries production (metric tons)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>ER.FSH.PROD.MT</td>\n",
       "      <td>Total fisheries production (metric tons)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>ER.H2O.FWTL.K3</td>\n",
       "      <td>Annual freshwater withdrawals, total (billion cubic meters)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>ER.H2O.INTR.K3</td>\n",
       "      <td>Renewable internal freshwater resources, total (billion cubic meters)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>IE.PPI.ENGY.CD</td>\n",
       "      <td>Investment in energy with private participation (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>IE.PPI.ICTI.CD</td>\n",
       "      <td>Investment in ICT with private participation (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>IE.PPI.TRAN.CD</td>\n",
       "      <td>Investment in transport with private participation (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>IE.PPI.WATR.CD</td>\n",
       "      <td>Investment in water and sanitation with private participation (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>IE.PPN.ENGY.CD</td>\n",
       "      <td>Public private partnerships investment in energy (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>IE.PPN.ICTI.CD</td>\n",
       "      <td>Public private partnerships investment in ICT (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>IE.PPN.TRAN.CD</td>\n",
       "      <td>Public private partnerships investment in transport (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>IE.PPN.WATR.CD</td>\n",
       "      <td>Public private partnerships investment in water and sanitation (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>IP.IDS.NRCT</td>\n",
       "      <td>Industrial design applications, nonresident, by count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>IP.IDS.RSCT</td>\n",
       "      <td>Industrial design applications, resident, by count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>IP.PAT.NRES</td>\n",
       "      <td>Patent applications, nonresidents</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>IP.PAT.RESD</td>\n",
       "      <td>Patent applications, residents</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>IP.TMK.NRCT</td>\n",
       "      <td>Trademark applications, nonresident, by count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>IP.TMK.RSCT</td>\n",
       "      <td>Trademark applications, resident, by count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>IS.AIR.DPRT</td>\n",
       "      <td>Air transport, registered carrier departures worldwide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>IS.AIR.GOOD.MT.K1</td>\n",
       "      <td>Air transport, freight (million ton-km)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>IS.AIR.PSGR</td>\n",
       "      <td>Air transport, passengers carried</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>IS.SHP.GOOD.TU</td>\n",
       "      <td>Container port traffic (TEU: 20 foot equivalent units)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>IT.CEL.SETS</td>\n",
       "      <td>Mobile cellular subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>IT.MLT.MAIN</td>\n",
       "      <td>Fixed telephone subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>Fixed broadband subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>IT.NET.SECR</td>\n",
       "      <td>Secure Internet servers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>MS.MIL.MPRT.KD</td>\n",
       "      <td>Arms imports (SIPRI trend indicator values)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>MS.MIL.TOTL.P1</td>\n",
       "      <td>Armed forces personnel, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>MS.MIL.XPRT.KD</td>\n",
       "      <td>Arms exports (SIPRI trend indicator values)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>SE.PRM.ENRL</td>\n",
       "      <td>Primary education, pupils</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>SE.PRM.TCHR</td>\n",
       "      <td>Primary education, teachers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>SE.PRM.UNER</td>\n",
       "      <td>Children out of school, primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>SE.PRM.UNER.FE</td>\n",
       "      <td>Children out of school, primary, female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>SE.PRM.UNER.MA</td>\n",
       "      <td>Children out of school, primary, male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>SE.SEC.ENRL</td>\n",
       "      <td>Secondary education, pupils</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>SE.SEC.ENRL.GC</td>\n",
       "      <td>Secondary education, general pupils</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>SE.SEC.ENRL.VO</td>\n",
       "      <td>Secondary education, vocational pupils</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>SE.SEC.TCHR</td>\n",
       "      <td>Secondary education, teachers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>SE.SEC.TCHR.FE</td>\n",
       "      <td>Secondary education, teachers, female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>SH.DTH.0509</td>\n",
       "      <td>Number of deaths ages 5-9 years</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>SH.DTH.1014</td>\n",
       "      <td>Number of deaths ages 10-14 years</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>SH.DTH.1519</td>\n",
       "      <td>Number of deaths ages 15-19 years</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>SH.DTH.2024</td>\n",
       "      <td>Number of deaths ages 20-24 years</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>SH.DTH.IMRT</td>\n",
       "      <td>Number of infant deaths</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>SH.DTH.MORT</td>\n",
       "      <td>Number of under-five deaths</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>SH.DTH.NMRT</td>\n",
       "      <td>Number of neonatal deaths</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>SH.MMR.DTHS</td>\n",
       "      <td>Number of maternal deaths</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>SH.UHC.NOP1.TO</td>\n",
       "      <td>Number of people pushed below the $1.90 ($ 2011 PPP) poverty line by out-of-pocket health care expenditure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>SH.UHC.NOP2.TO</td>\n",
       "      <td>Number of people pushed below the $3.20 ($ 2011 PPP) poverty line by out-of-pocket health care expenditure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>SH.UHC.OOPC.10.TO</td>\n",
       "      <td>Number of people spending more than 10% of household consumption or income on out-of-pocket health care expenditure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>SH.UHC.OOPC.25.TO</td>\n",
       "      <td>Number of people spending more than 25% of household consumption or income on out-of-pocket health care expenditure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>SL.TLF.TOTL.IN</td>\n",
       "      <td>Labor force, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>SM.POP.NETM</td>\n",
       "      <td>Net migration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>SM.POP.REFG</td>\n",
       "      <td>Refugee population by country or territory of asylum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>SM.POP.REFG.OR</td>\n",
       "      <td>Refugee population by country or territory of origin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>SM.POP.TOTL</td>\n",
       "      <td>International migrant stock, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>SP.POP.0014.FE.IN</td>\n",
       "      <td>Population ages 0-14, female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>SP.POP.0014.MA.IN</td>\n",
       "      <td>Population ages 0-14, male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>SP.POP.0014.TO</td>\n",
       "      <td>Population ages 0-14, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>SP.POP.1564.FE.IN</td>\n",
       "      <td>Population ages 15-64, female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>SP.POP.1564.MA.IN</td>\n",
       "      <td>Population ages 15-64, male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>SP.POP.1564.TO</td>\n",
       "      <td>Population ages 15-64, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>SP.POP.65UP.FE.IN</td>\n",
       "      <td>Population ages 65 and above, female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>SP.POP.65UP.MA.IN</td>\n",
       "      <td>Population ages 65 and above, male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>SP.POP.65UP.TO</td>\n",
       "      <td>Population ages 65 and above, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>SP.POP.TOTL</td>\n",
       "      <td>Population, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>SP.POP.TOTL.FE.IN</td>\n",
       "      <td>Population, female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>SP.POP.TOTL.MA.IN</td>\n",
       "      <td>Population, male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>SP.RUR.TOTL</td>\n",
       "      <td>Rural population</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>SP.URB.TOTL</td>\n",
       "      <td>Urban population</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>TM.VAL.MRCH.WL.CD</td>\n",
       "      <td>Merchandise imports by the reporting economy (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>TX.VAL.MRCH.WL.CD</td>\n",
       "      <td>Merchandise exports by the reporting economy (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>TX.VAL.TECH.CD</td>\n",
       "      <td>High-technology exports (current US$)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>VC.BTL.DETH</td>\n",
       "      <td>Battle-related deaths (number of people)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>VC.IDP.TOCV</td>\n",
       "      <td>Internally displaced persons, total displaced by conflict and violence (number of people)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           indicator_code  \\\n",
       "0          AG.AGR.TRAC.NO   \n",
       "1          AG.LND.AGRI.K2   \n",
       "2          AG.LND.CREL.HA   \n",
       "3       AG.LND.EL5M.RU.K2   \n",
       "4       AG.LND.EL5M.UR.K2   \n",
       "5          AG.LND.FRST.K2   \n",
       "6          AG.LND.TOTL.K2   \n",
       "7       AG.LND.TOTL.RU.K2   \n",
       "8       AG.LND.TOTL.UR.K2   \n",
       "9          AG.PRD.CREL.MT   \n",
       "10         AG.SRF.TOTL.K2   \n",
       "11         BM.GSR.FCTY.CD   \n",
       "12         BM.GSR.ROYL.CD   \n",
       "13      BM.KLT.DINV.CD.WD   \n",
       "14         BM.TRF.PRVT.CD   \n",
       "15      BM.TRF.PWKR.CD.DT   \n",
       "16      BX.GRT.EXTA.CD.WD   \n",
       "17      BX.GRT.TECH.CD.WD   \n",
       "18         BX.GSR.CCIS.CD   \n",
       "19         BX.GSR.FCTY.CD   \n",
       "20         BX.GSR.ROYL.CD   \n",
       "21      BX.KLT.DINV.CD.WD   \n",
       "22      BX.PEF.TOTL.CD.WD   \n",
       "23         BX.TRF.CURR.CD   \n",
       "24         BX.TRF.PWKR.CD   \n",
       "25      BX.TRF.PWKR.CD.DT   \n",
       "26         CM.MKT.LCAP.CD   \n",
       "27         CM.MKT.LDOM.NO   \n",
       "28         CM.MKT.TRAD.CD   \n",
       "29         DC.DAC.AUSL.CD   \n",
       "30         DC.DAC.AUTL.CD   \n",
       "31         DC.DAC.BELL.CD   \n",
       "32         DC.DAC.CANL.CD   \n",
       "33         DC.DAC.CECL.CD   \n",
       "34         DC.DAC.CHEL.CD   \n",
       "35         DC.DAC.CZEL.CD   \n",
       "36         DC.DAC.DEUL.CD   \n",
       "37         DC.DAC.DNKL.CD   \n",
       "38         DC.DAC.ESPL.CD   \n",
       "39         DC.DAC.FINL.CD   \n",
       "40         DC.DAC.FRAL.CD   \n",
       "41         DC.DAC.GBRL.CD   \n",
       "42         DC.DAC.GRCL.CD   \n",
       "43         DC.DAC.HUNL.CD   \n",
       "44         DC.DAC.IRLL.CD   \n",
       "45         DC.DAC.ISLL.CD   \n",
       "46         DC.DAC.ITAL.CD   \n",
       "47         DC.DAC.JPNL.CD   \n",
       "48         DC.DAC.KORL.CD   \n",
       "49         DC.DAC.LUXL.CD   \n",
       "50         DC.DAC.NLDL.CD   \n",
       "51         DC.DAC.NORL.CD   \n",
       "52         DC.DAC.NZLL.CD   \n",
       "53         DC.DAC.POLL.CD   \n",
       "54         DC.DAC.PRTL.CD   \n",
       "55         DC.DAC.SVKL.CD   \n",
       "56         DC.DAC.SVNL.CD   \n",
       "57         DC.DAC.SWEL.CD   \n",
       "58         DC.DAC.TOTL.CD   \n",
       "59         DC.DAC.USAL.CD   \n",
       "60         DT.DOD.DECT.CD   \n",
       "61         DT.DOD.DIMF.CD   \n",
       "62         DT.DOD.DLXF.CD   \n",
       "63         DT.DOD.DPNG.CD   \n",
       "64         DT.DOD.DPPG.CD   \n",
       "65         DT.DOD.DSTC.CD   \n",
       "66         DT.DOD.MIBR.CD   \n",
       "67         DT.DOD.MIDA.CD   \n",
       "68         DT.DOD.MWBG.CD   \n",
       "69         DT.DOD.PVLX.CD   \n",
       "70         DT.NFL.BLAT.CD   \n",
       "71         DT.NFL.BOND.CD   \n",
       "72         DT.NFL.DPNG.CD   \n",
       "73         DT.NFL.FAOG.CD   \n",
       "74         DT.NFL.IAEA.CD   \n",
       "75         DT.NFL.IFAD.CD   \n",
       "76         DT.NFL.ILOG.CD   \n",
       "77         DT.NFL.IMFC.CD   \n",
       "78         DT.NFL.IMFN.CD   \n",
       "79         DT.NFL.MIBR.CD   \n",
       "80         DT.NFL.MIDA.CD   \n",
       "81         DT.NFL.MLAT.CD   \n",
       "82         DT.NFL.MOTH.CD   \n",
       "83         DT.NFL.NIFC.CD   \n",
       "84         DT.NFL.OFFT.CD   \n",
       "85         DT.NFL.PBND.CD   \n",
       "86         DT.NFL.PCBK.CD   \n",
       "87         DT.NFL.PCBO.CD   \n",
       "88         DT.NFL.PNGB.CD   \n",
       "89         DT.NFL.PNGC.CD   \n",
       "90         DT.NFL.PROP.CD   \n",
       "91         DT.NFL.PRVT.CD   \n",
       "92         DT.NFL.RDBC.CD   \n",
       "93         DT.NFL.RDBN.CD   \n",
       "94         DT.NFL.UNAI.CD   \n",
       "95         DT.NFL.UNCF.CD   \n",
       "96         DT.NFL.UNCR.CD   \n",
       "97         DT.NFL.UNDP.CD   \n",
       "98         DT.NFL.UNEC.CD   \n",
       "99         DT.NFL.UNEP.CD   \n",
       "100        DT.NFL.UNFP.CD   \n",
       "101        DT.NFL.UNID.CD   \n",
       "102        DT.NFL.UNPB.CD   \n",
       "103        DT.NFL.UNRW.CD   \n",
       "104        DT.NFL.UNTA.CD   \n",
       "105        DT.NFL.UNWT.CD   \n",
       "106        DT.NFL.WFPG.CD   \n",
       "107        DT.NFL.WHOL.CD   \n",
       "108        DT.ODA.ALLD.CD   \n",
       "109        DT.ODA.ALLD.KD   \n",
       "110        DT.ODA.OATL.CD   \n",
       "111        DT.ODA.OATL.KD   \n",
       "112        DT.ODA.ODAT.CD   \n",
       "113        DT.ODA.ODAT.KD   \n",
       "114        DT.TDS.DECT.CD   \n",
       "115        DT.TDS.DIMF.CD   \n",
       "116        DT.TDS.DPPG.CD   \n",
       "117        DT.TDS.MLAT.CD   \n",
       "118        EG.ELC.RNWX.KH   \n",
       "119     EN.ATM.GHGO.KT.CE   \n",
       "120     EN.ATM.GHGT.KT.CE   \n",
       "121  EN.ATM.METH.AG.KT.CE   \n",
       "122  EN.ATM.METH.EG.KT.CE   \n",
       "123     EN.ATM.METH.KT.CE   \n",
       "124  EN.ATM.NOXE.AG.KT.CE   \n",
       "125  EN.ATM.NOXE.EG.KT.CE   \n",
       "126     EN.ATM.NOXE.KT.CE   \n",
       "127        EN.BIR.THRD.NO   \n",
       "128        EN.FSH.THRD.NO   \n",
       "129        EN.HPT.THRD.NO   \n",
       "130        EN.MAM.THRD.NO   \n",
       "131        ER.FSH.AQUA.MT   \n",
       "132        ER.FSH.CAPT.MT   \n",
       "133        ER.FSH.PROD.MT   \n",
       "134        ER.H2O.FWTL.K3   \n",
       "135        ER.H2O.INTR.K3   \n",
       "136        IE.PPI.ENGY.CD   \n",
       "137        IE.PPI.ICTI.CD   \n",
       "138        IE.PPI.TRAN.CD   \n",
       "139        IE.PPI.WATR.CD   \n",
       "140        IE.PPN.ENGY.CD   \n",
       "141        IE.PPN.ICTI.CD   \n",
       "142        IE.PPN.TRAN.CD   \n",
       "143        IE.PPN.WATR.CD   \n",
       "144           IP.IDS.NRCT   \n",
       "145           IP.IDS.RSCT   \n",
       "146           IP.PAT.NRES   \n",
       "147           IP.PAT.RESD   \n",
       "148           IP.TMK.NRCT   \n",
       "149           IP.TMK.RSCT   \n",
       "150           IS.AIR.DPRT   \n",
       "151     IS.AIR.GOOD.MT.K1   \n",
       "152           IS.AIR.PSGR   \n",
       "153        IS.SHP.GOOD.TU   \n",
       "154           IT.CEL.SETS   \n",
       "155           IT.MLT.MAIN   \n",
       "156           IT.NET.BBND   \n",
       "157           IT.NET.SECR   \n",
       "158        MS.MIL.MPRT.KD   \n",
       "159        MS.MIL.TOTL.P1   \n",
       "160        MS.MIL.XPRT.KD   \n",
       "161           SE.PRM.ENRL   \n",
       "162           SE.PRM.TCHR   \n",
       "163           SE.PRM.UNER   \n",
       "164        SE.PRM.UNER.FE   \n",
       "165        SE.PRM.UNER.MA   \n",
       "166           SE.SEC.ENRL   \n",
       "167        SE.SEC.ENRL.GC   \n",
       "168        SE.SEC.ENRL.VO   \n",
       "169           SE.SEC.TCHR   \n",
       "170        SE.SEC.TCHR.FE   \n",
       "171           SH.DTH.0509   \n",
       "172           SH.DTH.1014   \n",
       "173           SH.DTH.1519   \n",
       "174           SH.DTH.2024   \n",
       "175           SH.DTH.IMRT   \n",
       "176           SH.DTH.MORT   \n",
       "177           SH.DTH.NMRT   \n",
       "178           SH.MMR.DTHS   \n",
       "179        SH.UHC.NOP1.TO   \n",
       "180        SH.UHC.NOP2.TO   \n",
       "181     SH.UHC.OOPC.10.TO   \n",
       "182     SH.UHC.OOPC.25.TO   \n",
       "183        SL.TLF.TOTL.IN   \n",
       "184           SM.POP.NETM   \n",
       "185           SM.POP.REFG   \n",
       "186        SM.POP.REFG.OR   \n",
       "187           SM.POP.TOTL   \n",
       "188     SP.POP.0014.FE.IN   \n",
       "189     SP.POP.0014.MA.IN   \n",
       "190        SP.POP.0014.TO   \n",
       "191     SP.POP.1564.FE.IN   \n",
       "192     SP.POP.1564.MA.IN   \n",
       "193        SP.POP.1564.TO   \n",
       "194     SP.POP.65UP.FE.IN   \n",
       "195     SP.POP.65UP.MA.IN   \n",
       "196        SP.POP.65UP.TO   \n",
       "197           SP.POP.TOTL   \n",
       "198     SP.POP.TOTL.FE.IN   \n",
       "199     SP.POP.TOTL.MA.IN   \n",
       "200           SP.RUR.TOTL   \n",
       "201           SP.URB.TOTL   \n",
       "202     TM.VAL.MRCH.WL.CD   \n",
       "203     TX.VAL.MRCH.WL.CD   \n",
       "204        TX.VAL.TECH.CD   \n",
       "205           VC.BTL.DETH   \n",
       "206           VC.IDP.TOCV   \n",
       "\n",
       "                                                                                                          indicator_name  \n",
       "0                                                                                       Agricultural machinery, tractors  \n",
       "1                                                                                             Agricultural land (sq. km)  \n",
       "2                                                                                Land under cereal production (hectares)  \n",
       "3                                                             Rural land area where elevation is below 5 meters (sq. km)  \n",
       "4                                                             Urban land area where elevation is below 5 meters (sq. km)  \n",
       "5                                                                                                   Forest area (sq. km)  \n",
       "6                                                                                                     Land area (sq. km)  \n",
       "7                                                                                               Rural land area (sq. km)  \n",
       "8                                                                                               Urban land area (sq. km)  \n",
       "9                                                                                        Cereal production (metric tons)  \n",
       "10                                                                                                 Surface area (sq. km)  \n",
       "11                                                                            Primary income payments (BoP, current US$)  \n",
       "12                                             Charges for the use of intellectual property, payments (BoP, current US$)  \n",
       "13                                                            Foreign direct investment, net outflows (BoP, current US$)  \n",
       "14                                                          Secondary income, other sectors, payments (BoP, current US$)  \n",
       "15                                                                              Personal remittances, paid (current US$)  \n",
       "16                                                            Grants, excluding technical cooperation (BoP, current US$)  \n",
       "17                                                                       Technical cooperation grants (BoP, current US$)  \n",
       "18                                                                                ICT service exports (BoP, current US$)  \n",
       "19                                                                            Primary income receipts (BoP, current US$)  \n",
       "20                                             Charges for the use of intellectual property, receipts (BoP, current US$)  \n",
       "21                                                             Foreign direct investment, net inflows (BoP, current US$)  \n",
       "22                                                                      Portfolio equity, net inflows (BoP, current US$)  \n",
       "23                                                                          Secondary income receipts (BoP, current US$)  \n",
       "24                                                                       Personal transfers, receipts (BoP, current US$)  \n",
       "25                                                                          Personal remittances, received (current US$)  \n",
       "26                                                      Market capitalization of listed domestic companies (current US$)  \n",
       "27                                                                                      Listed domestic companies, total  \n",
       "28                                                                              Stocks traded, total value (current US$)  \n",
       "29                                                      Net bilateral aid flows from DAC donors, Australia (current US$)  \n",
       "30                                                        Net bilateral aid flows from DAC donors, Austria (current US$)  \n",
       "31                                                        Net bilateral aid flows from DAC donors, Belgium (current US$)  \n",
       "32                                                         Net bilateral aid flows from DAC donors, Canada (current US$)  \n",
       "33                                    Net bilateral aid flows from DAC donors, European Union institutions (current US$)  \n",
       "34                                                    Net bilateral aid flows from DAC donors, Switzerland (current US$)  \n",
       "35                                                 Net bilateral aid flows from DAC donors, Czech Republic (current US$)  \n",
       "36                                                        Net bilateral aid flows from DAC donors, Germany (current US$)  \n",
       "37                                                        Net bilateral aid flows from DAC donors, Denmark (current US$)  \n",
       "38                                                          Net bilateral aid flows from DAC donors, Spain (current US$)  \n",
       "39                                                        Net bilateral aid flows from DAC donors, Finland (current US$)  \n",
       "40                                                         Net bilateral aid flows from DAC donors, France (current US$)  \n",
       "41                                                 Net bilateral aid flows from DAC donors, United Kingdom (current US$)  \n",
       "42                                                         Net bilateral aid flows from DAC donors, Greece (current US$)  \n",
       "43                                                        Net bilateral aid flows from DAC donors, Hungary (current US$)  \n",
       "44                                                        Net bilateral aid flows from DAC donors, Ireland (current US$)  \n",
       "45                                                        Net bilateral aid flows from DAC donors, Iceland (current US$)  \n",
       "46                                                          Net bilateral aid flows from DAC donors, Italy (current US$)  \n",
       "47                                                          Net bilateral aid flows from DAC donors, Japan (current US$)  \n",
       "48                                                    Net bilateral aid flows from DAC donors, Korea, Rep. (current US$)  \n",
       "49                                                     Net bilateral aid flows from DAC donors, Luxembourg (current US$)  \n",
       "50                                                    Net bilateral aid flows from DAC donors, Netherlands (current US$)  \n",
       "51                                                         Net bilateral aid flows from DAC donors, Norway (current US$)  \n",
       "52                                                    Net bilateral aid flows from DAC donors, New Zealand (current US$)  \n",
       "53                                                         Net bilateral aid flows from DAC donors, Poland (current US$)  \n",
       "54                                                       Net bilateral aid flows from DAC donors, Portugal (current US$)  \n",
       "55                                                Net bilateral aid flows from DAC donors, Slovak Republic (current US$)  \n",
       "56                                                       Net bilateral aid flows from DAC donors, Slovenia (current US$)  \n",
       "57                                                         Net bilateral aid flows from DAC donors, Sweden (current US$)  \n",
       "58                                                          Net bilateral aid flows from DAC donors, Total (current US$)  \n",
       "59                                                  Net bilateral aid flows from DAC donors, United States (current US$)  \n",
       "60                                                                        External debt stocks, total (DOD, current US$)  \n",
       "61                                                                                  Use of IMF credit (DOD, current US$)  \n",
       "62                                                                    External debt stocks, long-term (DOD, current US$)  \n",
       "63                                                  External debt stocks, private nonguaranteed (PNG) (DOD, current US$)  \n",
       "64                                         External debt stocks, public and publicly guaranteed (PPG) (DOD, current US$)  \n",
       "65                                                                   External debt stocks, short-term (DOD, current US$)  \n",
       "66                                                                                          PPG, IBRD (DOD, current US$)  \n",
       "67                                                                                           PPG, IDA (DOD, current US$)  \n",
       "68                                                                         IBRD loans and IDA credits (DOD, current US$)  \n",
       "69                                                                          Present value of external debt (current US$)  \n",
       "70                                                                     Net financial flows, bilateral (NFL, current US$)  \n",
       "71                                                            Portfolio investment, bonds (PPG + PNG) (NFL, current US$)  \n",
       "72                                            Net flows on external debt, private nonguaranteed (PNG) (NFL, current US$)  \n",
       "73                                                                Net official flows from UN agencies, FAO (current US$)  \n",
       "74                                                               Net official flows from UN agencies, IAEA (current US$)  \n",
       "75                                                               Net official flows from UN agencies, IFAD (current US$)  \n",
       "76                                                                Net official flows from UN agencies, ILO (current US$)  \n",
       "77                                                              Net financial flows, IMF concessional (NFL, current US$)  \n",
       "78                                                           Net financial flows, IMF nonconcessional (NFL, current US$)  \n",
       "79                                                                          Net financial flows, IBRD (NFL, current US$)  \n",
       "80                                                                           Net financial flows, IDA (NFL, current US$)  \n",
       "81                                                                  Net financial flows, multilateral (NFL, current US$)  \n",
       "82                                                                        Net financial flows, others (NFL, current US$)  \n",
       "83                                                                         IFC, private nonguaranteed (NFL, current US$)  \n",
       "84                                                                            PPG, official creditors (NFL, current US$)  \n",
       "85                                                                                         PPG, bonds (NFL, current US$)  \n",
       "86                                                                              PPG, commercial banks (NFL, current US$)  \n",
       "87                                                     Commercial banks and other lending (PPG + PNG) (NFL, current US$)  \n",
       "88                                                                                         PNG, bonds (NFL, current US$)  \n",
       "89                                                          PNG, commercial banks and other creditors (NFL, current US$)  \n",
       "90                                                                       PPG, other private creditors (NFL, current US$)  \n",
       "91                                                                             PPG, private creditors (NFL, current US$)  \n",
       "92                                                              Net financial flows, RDB concessional (NFL, current US$)  \n",
       "93                                                           Net financial flows, RDB nonconcessional (NFL, current US$)  \n",
       "94                                                             Net official flows from UN agencies, UNAIDS (current US$)  \n",
       "95                                                             Net official flows from UN agencies, UNICEF (current US$)  \n",
       "96                                                              Net official flows from UN agencies, UNHCR (current US$)  \n",
       "97                                                               Net official flows from UN agencies, UNDP (current US$)  \n",
       "98                                                              Net official flows from UN agencies, UNECE (current US$)  \n",
       "99                                                               Net official flows from UN agencies, UNEP (current US$)  \n",
       "100                                                             Net official flows from UN agencies, UNFPA (current US$)  \n",
       "101                                                            Net official flows from UN agencies, UNIDIR (current US$)  \n",
       "102                                                             Net official flows from UN agencies, UNPBF (current US$)  \n",
       "103                                                             Net official flows from UN agencies, UNRWA (current US$)  \n",
       "104                                                              Net official flows from UN agencies, UNTA (current US$)  \n",
       "105                                                             Net official flows from UN agencies, UNWTO (current US$)  \n",
       "106                                                               Net official flows from UN agencies, WFP (current US$)  \n",
       "107                                                               Net official flows from UN agencies, WHO (current US$)  \n",
       "108                                          Net official development assistance and official aid received (current US$)  \n",
       "109                                    Net official development assistance and official aid received (constant 2020 US$)  \n",
       "110                                                                              Net official aid received (current US$)  \n",
       "111                                                                        Net official aid received (constant 2020 US$)  \n",
       "112                                                           Net official development assistance received (current US$)  \n",
       "113                                                     Net official development assistance received (constant 2020 US$)  \n",
       "114                                                              Debt service on external debt, total (TDS, current US$)  \n",
       "115                                                                       IMF repurchases and charges (TDS, current US$)  \n",
       "116                               Debt service on external debt, public and publicly guaranteed (PPG) (TDS, current US$)  \n",
       "117                                                                         Multilateral debt service (TDS, current US$)  \n",
       "118                                         Electricity production from renewable sources, excluding hydroelectric (kWh)  \n",
       "119                            Other greenhouse gas emissions, HFC, PFC and SF6 (thousand metric tons of CO2 equivalent)  \n",
       "120                                                                Total greenhouse gas emissions (kt of CO2 equivalent)  \n",
       "121                                              Agricultural methane emissions (thousand metric tons of CO2 equivalent)  \n",
       "122                                          Methane emissions in energy sector (thousand metric tons of CO2 equivalent)  \n",
       "123                                                                             Methane emissions (kt of CO2 equivalent)  \n",
       "124                                        Agricultural nitrous oxide emissions (thousand metric tons of CO2 equivalent)  \n",
       "125                                    Nitrous oxide emissions in energy sector (thousand metric tons of CO2 equivalent)  \n",
       "126                                                     Nitrous oxide emissions (thousand metric tons of CO2 equivalent)  \n",
       "127                                                                                             Bird species, threatened  \n",
       "128                                                                                             Fish species, threatened  \n",
       "129                                                                                   Plant species (higher), threatened  \n",
       "130                                                                                           Mammal species, threatened  \n",
       "131                                                                                 Aquaculture production (metric tons)  \n",
       "132                                                                           Capture fisheries production (metric tons)  \n",
       "133                                                                             Total fisheries production (metric tons)  \n",
       "134                                                          Annual freshwater withdrawals, total (billion cubic meters)  \n",
       "135                                                Renewable internal freshwater resources, total (billion cubic meters)  \n",
       "136                                                        Investment in energy with private participation (current US$)  \n",
       "137                                                           Investment in ICT with private participation (current US$)  \n",
       "138                                                     Investment in transport with private participation (current US$)  \n",
       "139                                          Investment in water and sanitation with private participation (current US$)  \n",
       "140                                                       Public private partnerships investment in energy (current US$)  \n",
       "141                                                          Public private partnerships investment in ICT (current US$)  \n",
       "142                                                    Public private partnerships investment in transport (current US$)  \n",
       "143                                         Public private partnerships investment in water and sanitation (current US$)  \n",
       "144                                                                Industrial design applications, nonresident, by count  \n",
       "145                                                                   Industrial design applications, resident, by count  \n",
       "146                                                                                    Patent applications, nonresidents  \n",
       "147                                                                                       Patent applications, residents  \n",
       "148                                                                        Trademark applications, nonresident, by count  \n",
       "149                                                                           Trademark applications, resident, by count  \n",
       "150                                                               Air transport, registered carrier departures worldwide  \n",
       "151                                                                              Air transport, freight (million ton-km)  \n",
       "152                                                                                    Air transport, passengers carried  \n",
       "153                                                               Container port traffic (TEU: 20 foot equivalent units)  \n",
       "154                                                                                        Mobile cellular subscriptions  \n",
       "155                                                                                        Fixed telephone subscriptions  \n",
       "156                                                                                        Fixed broadband subscriptions  \n",
       "157                                                                                              Secure Internet servers  \n",
       "158                                                                          Arms imports (SIPRI trend indicator values)  \n",
       "159                                                                                        Armed forces personnel, total  \n",
       "160                                                                          Arms exports (SIPRI trend indicator values)  \n",
       "161                                                                                            Primary education, pupils  \n",
       "162                                                                                          Primary education, teachers  \n",
       "163                                                                                      Children out of school, primary  \n",
       "164                                                                              Children out of school, primary, female  \n",
       "165                                                                                Children out of school, primary, male  \n",
       "166                                                                                          Secondary education, pupils  \n",
       "167                                                                                  Secondary education, general pupils  \n",
       "168                                                                               Secondary education, vocational pupils  \n",
       "169                                                                                        Secondary education, teachers  \n",
       "170                                                                                Secondary education, teachers, female  \n",
       "171                                                                                      Number of deaths ages 5-9 years  \n",
       "172                                                                                    Number of deaths ages 10-14 years  \n",
       "173                                                                                    Number of deaths ages 15-19 years  \n",
       "174                                                                                    Number of deaths ages 20-24 years  \n",
       "175                                                                                              Number of infant deaths  \n",
       "176                                                                                          Number of under-five deaths  \n",
       "177                                                                                            Number of neonatal deaths  \n",
       "178                                                                                            Number of maternal deaths  \n",
       "179           Number of people pushed below the $1.90 ($ 2011 PPP) poverty line by out-of-pocket health care expenditure  \n",
       "180           Number of people pushed below the $3.20 ($ 2011 PPP) poverty line by out-of-pocket health care expenditure  \n",
       "181  Number of people spending more than 10% of household consumption or income on out-of-pocket health care expenditure  \n",
       "182  Number of people spending more than 25% of household consumption or income on out-of-pocket health care expenditure  \n",
       "183                                                                                                   Labor force, total  \n",
       "184                                                                                                        Net migration  \n",
       "185                                                                 Refugee population by country or territory of asylum  \n",
       "186                                                                 Refugee population by country or territory of origin  \n",
       "187                                                                                   International migrant stock, total  \n",
       "188                                                                                         Population ages 0-14, female  \n",
       "189                                                                                           Population ages 0-14, male  \n",
       "190                                                                                          Population ages 0-14, total  \n",
       "191                                                                                        Population ages 15-64, female  \n",
       "192                                                                                          Population ages 15-64, male  \n",
       "193                                                                                         Population ages 15-64, total  \n",
       "194                                                                                 Population ages 65 and above, female  \n",
       "195                                                                                   Population ages 65 and above, male  \n",
       "196                                                                                  Population ages 65 and above, total  \n",
       "197                                                                                                    Population, total  \n",
       "198                                                                                                   Population, female  \n",
       "199                                                                                                     Population, male  \n",
       "200                                                                                                     Rural population  \n",
       "201                                                                                                     Urban population  \n",
       "202                                                           Merchandise imports by the reporting economy (current US$)  \n",
       "203                                                           Merchandise exports by the reporting economy (current US$)  \n",
       "204                                                                                High-technology exports (current US$)  \n",
       "205                                                                             Battle-related deaths (number of people)  \n",
       "206                            Internally displaced persons, total displaced by conflict and violence (number of people)  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Our dataset has multiple types of metrics. The only ones that we care about are simple aggregates.\n",
    "simpleAggIndicators = seriesNoDuplicates \\\n",
    "                    .filter(\"lower(Aggregation_method) = 'sum'\") \\\n",
    "                    .select(\"indicator_code\", \"indicator_name\") \\\n",
    "                    .orderBy(\"indicator_code\")\n",
    "                    \n",
    "showDF(simpleAggIndicators, limitRows=500, truncate = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "2211f72a-c9ab-4b71-86cb-1af12eb28c26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IT.CEL.SETS</td>\n",
       "      <td>Mobile cellular subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>Fixed broadband subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SP.POP.0014.TO</td>\n",
       "      <td>Population ages 0-14, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SP.POP.1564.TO</td>\n",
       "      <td>Population ages 15-64, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SP.POP.65UP.TO</td>\n",
       "      <td>Population ages 65 and above, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>SP.POP.TOTL</td>\n",
       "      <td>Population, total</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   indicator_code                       indicator_name\n",
       "0     IT.CEL.SETS        Mobile cellular subscriptions\n",
       "1     IT.NET.BBND        Fixed broadband subscriptions\n",
       "2  SP.POP.0014.TO          Population ages 0-14, total\n",
       "3  SP.POP.1564.TO         Population ages 15-64, total\n",
       "4  SP.POP.65UP.TO  Population ages 65 and above, total\n",
       "5     SP.POP.TOTL                    Population, total"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Only keep the indicators that are relevant to requirements i.e. Population indicators and Cellular and Broadband penetration\n",
    "targetIndicators = simpleAggIndicators.filter(\"lower(indicator_name) like '%population%total%'\" +\n",
    "                            \"or lower(indicator_name) like '%cellular%' \" +\n",
    "                            \"or lower(indicator_name) like '%broadband%' \")\n",
    "showDF(targetIndicators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d67ebbf8-594d-4856-9012-808d24e10e5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>1960</th>\n",
       "      <th>1961</th>\n",
       "      <th>1962</th>\n",
       "      <th>1963</th>\n",
       "      <th>1964</th>\n",
       "      <th>1965</th>\n",
       "      <th>1966</th>\n",
       "      <th>1967</th>\n",
       "      <th>...</th>\n",
       "      <th>2012</th>\n",
       "      <th>2013</th>\n",
       "      <th>2014</th>\n",
       "      <th>2015</th>\n",
       "      <th>2016</th>\n",
       "      <th>2017</th>\n",
       "      <th>2018</th>\n",
       "      <th>2019</th>\n",
       "      <th>2020</th>\n",
       "      <th>2021</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>EG.CFT.ACCS.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>16.559819</td>\n",
       "      <td>16.936004</td>\n",
       "      <td>17.337896</td>\n",
       "      <td>17.687093</td>\n",
       "      <td>18.140971</td>\n",
       "      <td>18.491344</td>\n",
       "      <td>18.825520</td>\n",
       "      <td>19.272212</td>\n",
       "      <td>19.628009</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>EG.CFT.ACCS.RU.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6.281667</td>\n",
       "      <td>6.499471</td>\n",
       "      <td>6.680066</td>\n",
       "      <td>6.859110</td>\n",
       "      <td>7.016238</td>\n",
       "      <td>7.180364</td>\n",
       "      <td>7.322294</td>\n",
       "      <td>7.517191</td>\n",
       "      <td>7.651598</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>EG.CFT.ACCS.UR.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>37.601816</td>\n",
       "      <td>37.855399</td>\n",
       "      <td>38.046781</td>\n",
       "      <td>38.326255</td>\n",
       "      <td>38.468426</td>\n",
       "      <td>38.670044</td>\n",
       "      <td>38.722783</td>\n",
       "      <td>38.927016</td>\n",
       "      <td>39.042839</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>EG.ELC.ACCS.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>31.844384</td>\n",
       "      <td>31.794160</td>\n",
       "      <td>32.001027</td>\n",
       "      <td>33.871910</td>\n",
       "      <td>38.880173</td>\n",
       "      <td>40.261358</td>\n",
       "      <td>43.061877</td>\n",
       "      <td>44.270860</td>\n",
       "      <td>45.803485</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>EG.ELC.ACCS.RU.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>19.402592</td>\n",
       "      <td>18.663502</td>\n",
       "      <td>17.633986</td>\n",
       "      <td>16.464681</td>\n",
       "      <td>24.531436</td>\n",
       "      <td>25.345111</td>\n",
       "      <td>27.449908</td>\n",
       "      <td>29.641760</td>\n",
       "      <td>30.404935</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>EG.ELC.ACCS.UR.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>66.945277</td>\n",
       "      <td>67.112206</td>\n",
       "      <td>66.283426</td>\n",
       "      <td>67.080235</td>\n",
       "      <td>69.132292</td>\n",
       "      <td>70.928567</td>\n",
       "      <td>71.866136</td>\n",
       "      <td>73.332842</td>\n",
       "      <td>73.942949</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.FE.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.MA.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.OL.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.40.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.PL.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.60.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.SO.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AFE</td>\n",
       "      <td>FX.OWN.TOTL.YG.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AFE</td>\n",
       "      <td>per_si_allsi.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AFE</td>\n",
       "      <td>per_allsp.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>AFE</td>\n",
       "      <td>per_sa_allsa.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AFE</td>\n",
       "      <td>per_lm_alllm.adq_pop_tot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>AFE</td>\n",
       "      <td>SE.PRM.TENR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code            indicator_code  1960  1961  1962  1963  1964  \\\n",
       "0              AFE            EG.CFT.ACCS.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "1              AFE         EG.CFT.ACCS.RU.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "2              AFE         EG.CFT.ACCS.UR.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "3              AFE            EG.ELC.ACCS.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "4              AFE         EG.ELC.ACCS.RU.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "5              AFE         EG.ELC.ACCS.UR.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "6              AFE            FX.OWN.TOTL.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "7              AFE         FX.OWN.TOTL.FE.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "8              AFE         FX.OWN.TOTL.MA.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "9              AFE         FX.OWN.TOTL.OL.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "10             AFE         FX.OWN.TOTL.40.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "11             AFE         FX.OWN.TOTL.PL.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "12             AFE         FX.OWN.TOTL.60.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "13             AFE         FX.OWN.TOTL.SO.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "14             AFE         FX.OWN.TOTL.YG.ZS   NaN   NaN   NaN   NaN   NaN   \n",
       "15             AFE  per_si_allsi.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   \n",
       "16             AFE     per_allsp.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   \n",
       "17             AFE  per_sa_allsa.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   \n",
       "18             AFE  per_lm_alllm.adq_pop_tot   NaN   NaN   NaN   NaN   NaN   \n",
       "19             AFE               SE.PRM.TENR   NaN   NaN   NaN   NaN   NaN   \n",
       "\n",
       "    1965  1966  1967  ...       2012       2013       2014       2015  \\\n",
       "0    NaN   NaN   NaN  ...  16.559819  16.936004  17.337896  17.687093   \n",
       "1    NaN   NaN   NaN  ...   6.281667   6.499471   6.680066   6.859110   \n",
       "2    NaN   NaN   NaN  ...  37.601816  37.855399  38.046781  38.326255   \n",
       "3    NaN   NaN   NaN  ...  31.844384  31.794160  32.001027  33.871910   \n",
       "4    NaN   NaN   NaN  ...  19.402592  18.663502  17.633986  16.464681   \n",
       "5    NaN   NaN   NaN  ...  66.945277  67.112206  66.283426  67.080235   \n",
       "6    NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "7    NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "8    NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "9    NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "10   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "11   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "12   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "13   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "14   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "15   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "16   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "17   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "18   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "19   NaN   NaN   NaN  ...        NaN        NaN        NaN        NaN   \n",
       "\n",
       "         2016       2017       2018       2019       2020  2021  \n",
       "0   18.140971  18.491344  18.825520  19.272212  19.628009   NaN  \n",
       "1    7.016238   7.180364   7.322294   7.517191   7.651598   NaN  \n",
       "2   38.468426  38.670044  38.722783  38.927016  39.042839   NaN  \n",
       "3   38.880173  40.261358  43.061877  44.270860  45.803485   NaN  \n",
       "4   24.531436  25.345111  27.449908  29.641760  30.404935   NaN  \n",
       "5   69.132292  70.928567  71.866136  73.332842  73.942949   NaN  \n",
       "6         NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "7         NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "8         NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "9         NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "10        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "11        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "12        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "13        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "14        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "15        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "16        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "17        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "18        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "19        NaN        NaN        NaN        NaN        NaN   NaN  \n",
       "\n",
       "[20 rows x 64 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "383572"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indicatorsData = df_indicators \\\n",
    "                .withColumnRenamed(\"Indicator Code\", \"indicator_code\") \\\n",
    "                .withColumnRenamed(\"Country Code\", \"wb_country_code\") \\\n",
    "                .drop(\"Indicator Name\") \\\n",
    "                .drop(\"Country Name\") \n",
    "showDF(indicatorsData)\n",
    "indicatorsData.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b3baf4fb-374f-408d-aec2-a36202802a9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>1960</th>\n",
       "      <th>1961</th>\n",
       "      <th>1962</th>\n",
       "      <th>1963</th>\n",
       "      <th>1964</th>\n",
       "      <th>1965</th>\n",
       "      <th>1966</th>\n",
       "      <th>1967</th>\n",
       "      <th>...</th>\n",
       "      <th>2013</th>\n",
       "      <th>2014</th>\n",
       "      <th>2015</th>\n",
       "      <th>2016</th>\n",
       "      <th>2017</th>\n",
       "      <th>2018</th>\n",
       "      <th>2019</th>\n",
       "      <th>2020</th>\n",
       "      <th>2021</th>\n",
       "      <th>indicator_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2572220.0</td>\n",
       "      <td>3123787.0</td>\n",
       "      <td>3086933.0</td>\n",
       "      <td>3129499.0</td>\n",
       "      <td>3187389.0</td>\n",
       "      <td>3286161.0</td>\n",
       "      <td>4119216.0</td>\n",
       "      <td>4929182.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fixed broadband subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.CEL.SETS</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>327628287.0</td>\n",
       "      <td>361512328.0</td>\n",
       "      <td>403793199.0</td>\n",
       "      <td>396261205.0</td>\n",
       "      <td>406413955.0</td>\n",
       "      <td>386012231.0</td>\n",
       "      <td>458769966.0</td>\n",
       "      <td>486868958.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mobile cellular subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>SP.POP.0014.TO</td>\n",
       "      <td>57144288.0</td>\n",
       "      <td>58943932.0</td>\n",
       "      <td>60748348.0</td>\n",
       "      <td>62553938.0</td>\n",
       "      <td>64347711.0</td>\n",
       "      <td>66128065.0</td>\n",
       "      <td>68203382.0</td>\n",
       "      <td>70220874.0</td>\n",
       "      <td>...</td>\n",
       "      <td>241896989.0</td>\n",
       "      <td>247356893.0</td>\n",
       "      <td>252787720.0</td>\n",
       "      <td>258306434.0</td>\n",
       "      <td>263744012.0</td>\n",
       "      <td>269079146.0</td>\n",
       "      <td>274282381.0</td>\n",
       "      <td>279340111.0</td>\n",
       "      <td>284463145.0</td>\n",
       "      <td>Population ages 0-14, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>SP.POP.1564.TO</td>\n",
       "      <td>69648709.0</td>\n",
       "      <td>71064806.0</td>\n",
       "      <td>72614824.0</td>\n",
       "      <td>74300783.0</td>\n",
       "      <td>76131179.0</td>\n",
       "      <td>78107040.0</td>\n",
       "      <td>79886565.0</td>\n",
       "      <td>81867716.0</td>\n",
       "      <td>...</td>\n",
       "      <td>303709474.0</td>\n",
       "      <td>313182403.0</td>\n",
       "      <td>322989451.0</td>\n",
       "      <td>332917687.0</td>\n",
       "      <td>343225705.0</td>\n",
       "      <td>353905517.0</td>\n",
       "      <td>364955873.0</td>\n",
       "      <td>376366386.0</td>\n",
       "      <td>387857061.0</td>\n",
       "      <td>Population ages 15-64, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>SP.POP.65UP.TO</td>\n",
       "      <td>4043770.0</td>\n",
       "      <td>4151048.0</td>\n",
       "      <td>4251472.0</td>\n",
       "      <td>4347316.0</td>\n",
       "      <td>4441300.0</td>\n",
       "      <td>4534866.0</td>\n",
       "      <td>4662727.0</td>\n",
       "      <td>4787863.0</td>\n",
       "      <td>...</td>\n",
       "      <td>16995114.0</td>\n",
       "      <td>17536074.0</td>\n",
       "      <td>18094678.0</td>\n",
       "      <td>18754826.0</td>\n",
       "      <td>19423165.0</td>\n",
       "      <td>20105467.0</td>\n",
       "      <td>20808019.0</td>\n",
       "      <td>21536800.0</td>\n",
       "      <td>22344911.0</td>\n",
       "      <td>Population ages 65 and above, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>SP.POP.TOTL</td>\n",
       "      <td>130836765.0</td>\n",
       "      <td>134159786.0</td>\n",
       "      <td>137614644.0</td>\n",
       "      <td>141202036.0</td>\n",
       "      <td>144920186.0</td>\n",
       "      <td>148769974.0</td>\n",
       "      <td>152752671.0</td>\n",
       "      <td>156876454.0</td>\n",
       "      <td>...</td>\n",
       "      <td>562601578.0</td>\n",
       "      <td>578075373.0</td>\n",
       "      <td>593871847.0</td>\n",
       "      <td>609978946.0</td>\n",
       "      <td>626392880.0</td>\n",
       "      <td>643090131.0</td>\n",
       "      <td>660046272.0</td>\n",
       "      <td>677243299.0</td>\n",
       "      <td>694665117.0</td>\n",
       "      <td>Population, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>399466.0</td>\n",
       "      <td>489777.0</td>\n",
       "      <td>565867.0</td>\n",
       "      <td>971317.0</td>\n",
       "      <td>980402.0</td>\n",
       "      <td>1084105.0</td>\n",
       "      <td>1209936.0</td>\n",
       "      <td>1743931.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fixed broadband subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IT.CEL.SETS</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>290449890.0</td>\n",
       "      <td>319468155.0</td>\n",
       "      <td>348206501.0</td>\n",
       "      <td>356136490.0</td>\n",
       "      <td>358816819.0</td>\n",
       "      <td>384382192.0</td>\n",
       "      <td>422941609.0</td>\n",
       "      <td>457048002.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mobile cellular subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFW</td>\n",
       "      <td>SP.POP.0014.TO</td>\n",
       "      <td>40179920.0</td>\n",
       "      <td>41258443.0</td>\n",
       "      <td>42322255.0</td>\n",
       "      <td>43381248.0</td>\n",
       "      <td>44435635.0</td>\n",
       "      <td>45485151.0</td>\n",
       "      <td>46815458.0</td>\n",
       "      <td>48101375.0</td>\n",
       "      <td>...</td>\n",
       "      <td>168071948.0</td>\n",
       "      <td>172375593.0</td>\n",
       "      <td>176628546.0</td>\n",
       "      <td>181074105.0</td>\n",
       "      <td>185398355.0</td>\n",
       "      <td>189619289.0</td>\n",
       "      <td>193768241.0</td>\n",
       "      <td>197861564.0</td>\n",
       "      <td>202086389.0</td>\n",
       "      <td>Population ages 0-14, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFW</td>\n",
       "      <td>SP.POP.1564.TO</td>\n",
       "      <td>53387204.0</td>\n",
       "      <td>54235983.0</td>\n",
       "      <td>55195109.0</td>\n",
       "      <td>56251904.0</td>\n",
       "      <td>57401041.0</td>\n",
       "      <td>58639680.0</td>\n",
       "      <td>59627220.0</td>\n",
       "      <td>60751064.0</td>\n",
       "      <td>...</td>\n",
       "      <td>201761298.0</td>\n",
       "      <td>207655193.0</td>\n",
       "      <td>213860470.0</td>\n",
       "      <td>220036067.0</td>\n",
       "      <td>226593701.0</td>\n",
       "      <td>233498281.0</td>\n",
       "      <td>240697092.0</td>\n",
       "      <td>248156675.0</td>\n",
       "      <td>255592231.0</td>\n",
       "      <td>Population ages 15-64, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFW</td>\n",
       "      <td>SP.POP.65UP.TO</td>\n",
       "      <td>2829296.0</td>\n",
       "      <td>2912796.0</td>\n",
       "      <td>2989600.0</td>\n",
       "      <td>3058187.0</td>\n",
       "      <td>3116795.0</td>\n",
       "      <td>3165043.0</td>\n",
       "      <td>3259134.0</td>\n",
       "      <td>3343512.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10604651.0</td>\n",
       "      <td>10852195.0</td>\n",
       "      <td>11097632.0</td>\n",
       "      <td>11441125.0</td>\n",
       "      <td>11777875.0</td>\n",
       "      <td>12111811.0</td>\n",
       "      <td>12446267.0</td>\n",
       "      <td>12785236.0</td>\n",
       "      <td>13220250.0</td>\n",
       "      <td>Population ages 65 and above, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFW</td>\n",
       "      <td>SP.POP.TOTL</td>\n",
       "      <td>96396419.0</td>\n",
       "      <td>98407221.0</td>\n",
       "      <td>100506960.0</td>\n",
       "      <td>102691339.0</td>\n",
       "      <td>104953470.0</td>\n",
       "      <td>107289875.0</td>\n",
       "      <td>109701811.0</td>\n",
       "      <td>112195950.0</td>\n",
       "      <td>...</td>\n",
       "      <td>380437896.0</td>\n",
       "      <td>390882979.0</td>\n",
       "      <td>401586651.0</td>\n",
       "      <td>412551299.0</td>\n",
       "      <td>423769930.0</td>\n",
       "      <td>435229381.0</td>\n",
       "      <td>446911598.0</td>\n",
       "      <td>458803476.0</td>\n",
       "      <td>470898870.0</td>\n",
       "      <td>Population, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8880291.0</td>\n",
       "      <td>10801834.0</td>\n",
       "      <td>18081304.0</td>\n",
       "      <td>20019879.0</td>\n",
       "      <td>28218281.0</td>\n",
       "      <td>30067212.0</td>\n",
       "      <td>33114238.0</td>\n",
       "      <td>38898085.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fixed broadband subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IT.CEL.SETS</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>407704505.0</td>\n",
       "      <td>415029432.0</td>\n",
       "      <td>419013963.0</td>\n",
       "      <td>417212258.0</td>\n",
       "      <td>416484374.0</td>\n",
       "      <td>411117427.0</td>\n",
       "      <td>421459796.0</td>\n",
       "      <td>423344019.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mobile cellular subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ARB</td>\n",
       "      <td>SP.POP.0014.TO</td>\n",
       "      <td>39900270.0</td>\n",
       "      <td>41339716.0</td>\n",
       "      <td>42792876.0</td>\n",
       "      <td>44248953.0</td>\n",
       "      <td>45685299.0</td>\n",
       "      <td>47089854.0</td>\n",
       "      <td>48668390.0</td>\n",
       "      <td>50184667.0</td>\n",
       "      <td>...</td>\n",
       "      <td>126210619.0</td>\n",
       "      <td>128447163.0</td>\n",
       "      <td>130629545.0</td>\n",
       "      <td>133196474.0</td>\n",
       "      <td>135482326.0</td>\n",
       "      <td>137628016.0</td>\n",
       "      <td>139782894.0</td>\n",
       "      <td>142003455.0</td>\n",
       "      <td>144001243.0</td>\n",
       "      <td>Population ages 0-14, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ARB</td>\n",
       "      <td>SP.POP.1564.TO</td>\n",
       "      <td>49063221.0</td>\n",
       "      <td>50032214.0</td>\n",
       "      <td>51072091.0</td>\n",
       "      <td>52200464.0</td>\n",
       "      <td>53449221.0</td>\n",
       "      <td>54836640.0</td>\n",
       "      <td>56150923.0</td>\n",
       "      <td>57648452.0</td>\n",
       "      <td>...</td>\n",
       "      <td>237522658.0</td>\n",
       "      <td>242986682.0</td>\n",
       "      <td>248365389.0</td>\n",
       "      <td>253171312.0</td>\n",
       "      <td>258081365.0</td>\n",
       "      <td>263087505.0</td>\n",
       "      <td>268157877.0</td>\n",
       "      <td>273323616.0</td>\n",
       "      <td>278842504.0</td>\n",
       "      <td>Population ages 15-64, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ARB</td>\n",
       "      <td>SP.POP.65UP.TO</td>\n",
       "      <td>3234225.0</td>\n",
       "      <td>3352609.0</td>\n",
       "      <td>3469470.0</td>\n",
       "      <td>3584776.0</td>\n",
       "      <td>3698270.0</td>\n",
       "      <td>3809935.0</td>\n",
       "      <td>3939319.0</td>\n",
       "      <td>4066217.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15963198.0</td>\n",
       "      <td>16465991.0</td>\n",
       "      <td>17033363.0</td>\n",
       "      <td>17675107.0</td>\n",
       "      <td>18379135.0</td>\n",
       "      <td>19136464.0</td>\n",
       "      <td>19929502.0</td>\n",
       "      <td>20753658.0</td>\n",
       "      <td>21674036.0</td>\n",
       "      <td>Population ages 65 and above, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ARB</td>\n",
       "      <td>SP.POP.TOTL</td>\n",
       "      <td>92197715.0</td>\n",
       "      <td>94724540.0</td>\n",
       "      <td>97334438.0</td>\n",
       "      <td>100034191.0</td>\n",
       "      <td>102832792.0</td>\n",
       "      <td>105736428.0</td>\n",
       "      <td>108758634.0</td>\n",
       "      <td>111899335.0</td>\n",
       "      <td>...</td>\n",
       "      <td>379696477.0</td>\n",
       "      <td>387899835.0</td>\n",
       "      <td>396028301.0</td>\n",
       "      <td>404042892.0</td>\n",
       "      <td>411942825.0</td>\n",
       "      <td>419851989.0</td>\n",
       "      <td>427870273.0</td>\n",
       "      <td>436080728.0</td>\n",
       "      <td>444517783.0</td>\n",
       "      <td>Population, total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>CSS</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>674419.0</td>\n",
       "      <td>753368.0</td>\n",
       "      <td>897134.0</td>\n",
       "      <td>971854.0</td>\n",
       "      <td>1024247.0</td>\n",
       "      <td>1007098.0</td>\n",
       "      <td>1152226.0</td>\n",
       "      <td>1340377.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fixed broadband subscriptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>CSS</td>\n",
       "      <td>IT.CEL.SETS</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7738401.0</td>\n",
       "      <td>7952805.0</td>\n",
       "      <td>8158698.0</td>\n",
       "      <td>8483364.0</td>\n",
       "      <td>8225309.0</td>\n",
       "      <td>7218705.0</td>\n",
       "      <td>8476897.0</td>\n",
       "      <td>8401653.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mobile cellular subscriptions</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 65 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code  indicator_code         1960         1961         1962  \\\n",
       "0              AFE     IT.NET.BBND          NaN          NaN          NaN   \n",
       "1              AFE     IT.CEL.SETS          0.0          NaN          NaN   \n",
       "2              AFE  SP.POP.0014.TO   57144288.0   58943932.0   60748348.0   \n",
       "3              AFE  SP.POP.1564.TO   69648709.0   71064806.0   72614824.0   \n",
       "4              AFE  SP.POP.65UP.TO    4043770.0    4151048.0    4251472.0   \n",
       "5              AFE     SP.POP.TOTL  130836765.0  134159786.0  137614644.0   \n",
       "6              AFW     IT.NET.BBND          NaN          NaN          NaN   \n",
       "7              AFW     IT.CEL.SETS          0.0          NaN          NaN   \n",
       "8              AFW  SP.POP.0014.TO   40179920.0   41258443.0   42322255.0   \n",
       "9              AFW  SP.POP.1564.TO   53387204.0   54235983.0   55195109.0   \n",
       "10             AFW  SP.POP.65UP.TO    2829296.0    2912796.0    2989600.0   \n",
       "11             AFW     SP.POP.TOTL   96396419.0   98407221.0  100506960.0   \n",
       "12             ARB     IT.NET.BBND          NaN          NaN          NaN   \n",
       "13             ARB     IT.CEL.SETS          0.0          NaN          NaN   \n",
       "14             ARB  SP.POP.0014.TO   39900270.0   41339716.0   42792876.0   \n",
       "15             ARB  SP.POP.1564.TO   49063221.0   50032214.0   51072091.0   \n",
       "16             ARB  SP.POP.65UP.TO    3234225.0    3352609.0    3469470.0   \n",
       "17             ARB     SP.POP.TOTL   92197715.0   94724540.0   97334438.0   \n",
       "18             CSS     IT.NET.BBND          NaN          NaN          NaN   \n",
       "19             CSS     IT.CEL.SETS          0.0          NaN          NaN   \n",
       "\n",
       "           1963         1964         1965         1966         1967  ...  \\\n",
       "0           NaN          NaN          NaN          NaN          NaN  ...   \n",
       "1           NaN          NaN          0.0          NaN          NaN  ...   \n",
       "2    62553938.0   64347711.0   66128065.0   68203382.0   70220874.0  ...   \n",
       "3    74300783.0   76131179.0   78107040.0   79886565.0   81867716.0  ...   \n",
       "4     4347316.0    4441300.0    4534866.0    4662727.0    4787863.0  ...   \n",
       "5   141202036.0  144920186.0  148769974.0  152752671.0  156876454.0  ...   \n",
       "6           NaN          NaN          NaN          NaN          NaN  ...   \n",
       "7           NaN          NaN          0.0          NaN          NaN  ...   \n",
       "8    43381248.0   44435635.0   45485151.0   46815458.0   48101375.0  ...   \n",
       "9    56251904.0   57401041.0   58639680.0   59627220.0   60751064.0  ...   \n",
       "10    3058187.0    3116795.0    3165043.0    3259134.0    3343512.0  ...   \n",
       "11  102691339.0  104953470.0  107289875.0  109701811.0  112195950.0  ...   \n",
       "12          NaN          NaN          NaN          NaN          NaN  ...   \n",
       "13          NaN          NaN          0.0          NaN          NaN  ...   \n",
       "14   44248953.0   45685299.0   47089854.0   48668390.0   50184667.0  ...   \n",
       "15   52200464.0   53449221.0   54836640.0   56150923.0   57648452.0  ...   \n",
       "16    3584776.0    3698270.0    3809935.0    3939319.0    4066217.0  ...   \n",
       "17  100034191.0  102832792.0  105736428.0  108758634.0  111899335.0  ...   \n",
       "18          NaN          NaN          NaN          NaN          NaN  ...   \n",
       "19          NaN          NaN          0.0          NaN          NaN  ...   \n",
       "\n",
       "           2013         2014         2015         2016         2017  \\\n",
       "0     2572220.0    3123787.0    3086933.0    3129499.0    3187389.0   \n",
       "1   327628287.0  361512328.0  403793199.0  396261205.0  406413955.0   \n",
       "2   241896989.0  247356893.0  252787720.0  258306434.0  263744012.0   \n",
       "3   303709474.0  313182403.0  322989451.0  332917687.0  343225705.0   \n",
       "4    16995114.0   17536074.0   18094678.0   18754826.0   19423165.0   \n",
       "5   562601578.0  578075373.0  593871847.0  609978946.0  626392880.0   \n",
       "6      399466.0     489777.0     565867.0     971317.0     980402.0   \n",
       "7   290449890.0  319468155.0  348206501.0  356136490.0  358816819.0   \n",
       "8   168071948.0  172375593.0  176628546.0  181074105.0  185398355.0   \n",
       "9   201761298.0  207655193.0  213860470.0  220036067.0  226593701.0   \n",
       "10   10604651.0   10852195.0   11097632.0   11441125.0   11777875.0   \n",
       "11  380437896.0  390882979.0  401586651.0  412551299.0  423769930.0   \n",
       "12    8880291.0   10801834.0   18081304.0   20019879.0   28218281.0   \n",
       "13  407704505.0  415029432.0  419013963.0  417212258.0  416484374.0   \n",
       "14  126210619.0  128447163.0  130629545.0  133196474.0  135482326.0   \n",
       "15  237522658.0  242986682.0  248365389.0  253171312.0  258081365.0   \n",
       "16   15963198.0   16465991.0   17033363.0   17675107.0   18379135.0   \n",
       "17  379696477.0  387899835.0  396028301.0  404042892.0  411942825.0   \n",
       "18     674419.0     753368.0     897134.0     971854.0    1024247.0   \n",
       "19    7738401.0    7952805.0    8158698.0    8483364.0    8225309.0   \n",
       "\n",
       "           2018         2019         2020         2021  \\\n",
       "0     3286161.0    4119216.0    4929182.0          NaN   \n",
       "1   386012231.0  458769966.0  486868958.0          NaN   \n",
       "2   269079146.0  274282381.0  279340111.0  284463145.0   \n",
       "3   353905517.0  364955873.0  376366386.0  387857061.0   \n",
       "4    20105467.0   20808019.0   21536800.0   22344911.0   \n",
       "5   643090131.0  660046272.0  677243299.0  694665117.0   \n",
       "6     1084105.0    1209936.0    1743931.0          NaN   \n",
       "7   384382192.0  422941609.0  457048002.0          NaN   \n",
       "8   189619289.0  193768241.0  197861564.0  202086389.0   \n",
       "9   233498281.0  240697092.0  248156675.0  255592231.0   \n",
       "10   12111811.0   12446267.0   12785236.0   13220250.0   \n",
       "11  435229381.0  446911598.0  458803476.0  470898870.0   \n",
       "12   30067212.0   33114238.0   38898085.0          NaN   \n",
       "13  411117427.0  421459796.0  423344019.0          NaN   \n",
       "14  137628016.0  139782894.0  142003455.0  144001243.0   \n",
       "15  263087505.0  268157877.0  273323616.0  278842504.0   \n",
       "16   19136464.0   19929502.0   20753658.0   21674036.0   \n",
       "17  419851989.0  427870273.0  436080728.0  444517783.0   \n",
       "18    1007098.0    1152226.0    1340377.0          NaN   \n",
       "19    7218705.0    8476897.0    8401653.0          NaN   \n",
       "\n",
       "                         indicator_name  \n",
       "0         Fixed broadband subscriptions  \n",
       "1         Mobile cellular subscriptions  \n",
       "2           Population ages 0-14, total  \n",
       "3          Population ages 15-64, total  \n",
       "4   Population ages 65 and above, total  \n",
       "5                     Population, total  \n",
       "6         Fixed broadband subscriptions  \n",
       "7         Mobile cellular subscriptions  \n",
       "8           Population ages 0-14, total  \n",
       "9          Population ages 15-64, total  \n",
       "10  Population ages 65 and above, total  \n",
       "11                    Population, total  \n",
       "12        Fixed broadband subscriptions  \n",
       "13        Mobile cellular subscriptions  \n",
       "14          Population ages 0-14, total  \n",
       "15         Population ages 15-64, total  \n",
       "16  Population ages 65 and above, total  \n",
       "17                    Population, total  \n",
       "18        Fixed broadband subscriptions  \n",
       "19        Mobile cellular subscriptions  \n",
       "\n",
       "[20 rows x 65 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Keep only the indicators that we care about\n",
    "targetIndicatorsData = indicatorsData.join(targetIndicators, indicatorsData.indicator_code == targetIndicators.indicator_code).drop(targetIndicators.indicator_code)\n",
    "showDF(targetIndicatorsData)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71f3736e",
   "metadata": {},
   "source": [
    "\n",
    "Model the DataFrame to have a row for each value instead of a column, allowing easier future data augmentation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c03d26f3-7391-4556-b1be-bac60cb14f97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011', '2012', '2013', '2014', '2015', '2016', '2017', '2018', '2019', '2020', '2021']\n"
     ]
    }
   ],
   "source": [
    "yearList = [x for x in targetIndicatorsData.schema.names \\\n",
    "             if x != 'wb_country_code' and x != 'indicator_code' and x != 'indicator_name'] \n",
    "\n",
    "print(yearList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "9071fac8-9ee3-461a-bc1c-26690e5c5197",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>year</th>\n",
       "      <th>indicator_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1960</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1961</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1962</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1963</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1964</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1965</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1966</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1967</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1968</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1969</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1970</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1971</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1972</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1973</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1974</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1975</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1976</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1977</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1978</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>1979</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code indicator_code  year  indicator_value\n",
       "0              AFE    IT.NET.BBND  1960              NaN\n",
       "1              AFE    IT.NET.BBND  1961              NaN\n",
       "2              AFE    IT.NET.BBND  1962              NaN\n",
       "3              AFE    IT.NET.BBND  1963              NaN\n",
       "4              AFE    IT.NET.BBND  1964              NaN\n",
       "5              AFE    IT.NET.BBND  1965              NaN\n",
       "6              AFE    IT.NET.BBND  1966              NaN\n",
       "7              AFE    IT.NET.BBND  1967              NaN\n",
       "8              AFE    IT.NET.BBND  1968              NaN\n",
       "9              AFE    IT.NET.BBND  1969              NaN\n",
       "10             AFE    IT.NET.BBND  1970              NaN\n",
       "11             AFE    IT.NET.BBND  1971              NaN\n",
       "12             AFE    IT.NET.BBND  1972              NaN\n",
       "13             AFE    IT.NET.BBND  1973              NaN\n",
       "14             AFE    IT.NET.BBND  1974              NaN\n",
       "15             AFE    IT.NET.BBND  1975              NaN\n",
       "16             AFE    IT.NET.BBND  1976              NaN\n",
       "17             AFE    IT.NET.BBND  1977              NaN\n",
       "18             AFE    IT.NET.BBND  1978              NaN\n",
       "19             AFE    IT.NET.BBND  1979              NaN"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "exprs = [F.struct(F.lit(year).alias(\"year\"), F.col(year).alias(\"indicator_value\")) for year in yearList]\n",
    "tempDF = targetIndicatorsData.withColumn(\"years\", F.explode(F.array(*exprs)))\n",
    "indicatorsDF = tempDF.select(\n",
    "    \"wb_country_code\", \n",
    "    \"indicator_code\", \n",
    "    F.col(\"years.year\").alias(\"year\"), \n",
    "    F.col(\"years.indicator_value\").alias(\"indicator_value\")\n",
    ")\n",
    "showDF(indicatorsDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b2ed14a0-7369-4627-b9fe-6e559cbdcffa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>IT.CEL.SETS</th>\n",
       "      <th>IT.NET.BBND</th>\n",
       "      <th>SP.POP.0014.TO</th>\n",
       "      <th>SP.POP.1564.TO</th>\n",
       "      <th>SP.POP.65UP.TO</th>\n",
       "      <th>SP.POP.TOTL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1960</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.177090e+10</td>\n",
       "      <td>1.774023e+10</td>\n",
       "      <td>1.419879e+09</td>\n",
       "      <td>3.094538e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1961</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.198995e+10</td>\n",
       "      <td>1.789734e+10</td>\n",
       "      <td>1.451559e+09</td>\n",
       "      <td>3.135351e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.228409e+10</td>\n",
       "      <td>1.812713e+10</td>\n",
       "      <td>1.483848e+09</td>\n",
       "      <td>3.191012e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1963</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.262982e+10</td>\n",
       "      <td>1.843879e+10</td>\n",
       "      <td>1.517585e+09</td>\n",
       "      <td>3.260163e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.294934e+10</td>\n",
       "      <td>1.878377e+10</td>\n",
       "      <td>1.549776e+09</td>\n",
       "      <td>3.329871e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1965</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.323753e+10</td>\n",
       "      <td>1.917767e+10</td>\n",
       "      <td>1.581610e+09</td>\n",
       "      <td>3.401303e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1966</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.356844e+10</td>\n",
       "      <td>1.954489e+10</td>\n",
       "      <td>1.634962e+09</td>\n",
       "      <td>3.476492e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.384890e+10</td>\n",
       "      <td>1.996080e+10</td>\n",
       "      <td>1.688033e+09</td>\n",
       "      <td>3.551476e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.410320e+10</td>\n",
       "      <td>2.042098e+10</td>\n",
       "      <td>1.741251e+09</td>\n",
       "      <td>3.628284e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.435947e+10</td>\n",
       "      <td>2.090678e+10</td>\n",
       "      <td>1.794466e+09</td>\n",
       "      <td>3.707854e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.462541e+10</td>\n",
       "      <td>2.140082e+10</td>\n",
       "      <td>1.846993e+09</td>\n",
       "      <td>3.789143e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.492759e+10</td>\n",
       "      <td>2.187299e+10</td>\n",
       "      <td>1.911867e+09</td>\n",
       "      <td>3.873104e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1972</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.521347e+10</td>\n",
       "      <td>2.234891e+10</td>\n",
       "      <td>1.973903e+09</td>\n",
       "      <td>3.955530e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.548715e+10</td>\n",
       "      <td>2.283746e+10</td>\n",
       "      <td>2.034273e+09</td>\n",
       "      <td>4.037831e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1974</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.573819e+10</td>\n",
       "      <td>2.334452e+10</td>\n",
       "      <td>2.094073e+09</td>\n",
       "      <td>4.119663e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1975</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.595658e+10</td>\n",
       "      <td>2.387064e+10</td>\n",
       "      <td>2.153575e+09</td>\n",
       "      <td>4.200106e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.617454e+10</td>\n",
       "      <td>2.437656e+10</td>\n",
       "      <td>2.225683e+09</td>\n",
       "      <td>4.279746e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.636056e+10</td>\n",
       "      <td>2.490989e+10</td>\n",
       "      <td>2.297709e+09</td>\n",
       "      <td>4.358925e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1978</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.652624e+10</td>\n",
       "      <td>2.547885e+10</td>\n",
       "      <td>2.368255e+09</td>\n",
       "      <td>4.439484e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1979</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.668272e+10</td>\n",
       "      <td>2.608174e+10</td>\n",
       "      <td>2.433694e+09</td>\n",
       "      <td>4.522009e+10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    year  IT.CEL.SETS  IT.NET.BBND  SP.POP.0014.TO  SP.POP.1564.TO  \\\n",
       "0   1960          0.0          NaN    1.177090e+10    1.774023e+10   \n",
       "1   1961          NaN          NaN    1.198995e+10    1.789734e+10   \n",
       "2   1962          NaN          NaN    1.228409e+10    1.812713e+10   \n",
       "3   1963          NaN          NaN    1.262982e+10    1.843879e+10   \n",
       "4   1964          NaN          NaN    1.294934e+10    1.878377e+10   \n",
       "5   1965          0.0          NaN    1.323753e+10    1.917767e+10   \n",
       "6   1966          NaN          NaN    1.356844e+10    1.954489e+10   \n",
       "7   1967          NaN          NaN    1.384890e+10    1.996080e+10   \n",
       "8   1968          NaN          NaN    1.410320e+10    2.042098e+10   \n",
       "9   1969          NaN          NaN    1.435947e+10    2.090678e+10   \n",
       "10  1970          0.0          NaN    1.462541e+10    2.140082e+10   \n",
       "11  1971          NaN          NaN    1.492759e+10    2.187299e+10   \n",
       "12  1972          NaN          NaN    1.521347e+10    2.234891e+10   \n",
       "13  1973          NaN          NaN    1.548715e+10    2.283746e+10   \n",
       "14  1974          NaN          NaN    1.573819e+10    2.334452e+10   \n",
       "15  1975          0.0          NaN    1.595658e+10    2.387064e+10   \n",
       "16  1976          0.0          NaN    1.617454e+10    2.437656e+10   \n",
       "17  1977          0.0          NaN    1.636056e+10    2.490989e+10   \n",
       "18  1978          0.0          NaN    1.652624e+10    2.547885e+10   \n",
       "19  1979          0.0          NaN    1.668272e+10    2.608174e+10   \n",
       "\n",
       "    SP.POP.65UP.TO   SP.POP.TOTL  \n",
       "0     1.419879e+09  3.094538e+10  \n",
       "1     1.451559e+09  3.135351e+10  \n",
       "2     1.483848e+09  3.191012e+10  \n",
       "3     1.517585e+09  3.260163e+10  \n",
       "4     1.549776e+09  3.329871e+10  \n",
       "5     1.581610e+09  3.401303e+10  \n",
       "6     1.634962e+09  3.476492e+10  \n",
       "7     1.688033e+09  3.551476e+10  \n",
       "8     1.741251e+09  3.628284e+10  \n",
       "9     1.794466e+09  3.707854e+10  \n",
       "10    1.846993e+09  3.789143e+10  \n",
       "11    1.911867e+09  3.873104e+10  \n",
       "12    1.973903e+09  3.955530e+10  \n",
       "13    2.034273e+09  4.037831e+10  \n",
       "14    2.094073e+09  4.119663e+10  \n",
       "15    2.153575e+09  4.200106e+10  \n",
       "16    2.225683e+09  4.279746e+10  \n",
       "17    2.297709e+09  4.358925e+10  \n",
       "18    2.368255e+09  4.439484e+10  \n",
       "19    2.433694e+09  4.522009e+10  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "yearPivot = indicatorsDF.groupBy(\"year\").pivot(\"indicator_code\").sum('indicator_value')\n",
    "showDF(yearPivot.orderBy(\"year\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "77b295a7-4b77-4565-b25b-3f3b328f5950",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- year: string (nullable = false)\n",
      " |-- IT.CEL.SETS: double (nullable = true)\n",
      " |-- IT.NET.BBND: double (nullable = true)\n",
      " |-- SP.POP.0014.TO: double (nullable = true)\n",
      " |-- SP.POP.1564.TO: double (nullable = true)\n",
      " |-- SP.POP.65UP.TO: double (nullable = true)\n",
      " |-- SP.POP.TOTL: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "yearPivot.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5d6a3000-7e9f-4aa6-81ee-128753e87a92",
   "metadata": {},
   "outputs": [],
   "source": [
    "yearPivotDF = yearPivot.orderBy('year') \\\n",
    "    .withColumnRenamed('IT.CEL.SETS', 'cellular_subscriptions') \\\n",
    "    .withColumnRenamed('IT.NET.BBND', 'broadband_subscriptions') \\\n",
    "    .withColumnRenamed('SP.POP.0014.TO', 'population_age_0_to_14') \\\n",
    "    .withColumnRenamed('SP.POP.1564.TO', 'population_age_15_64') \\\n",
    "    .withColumnRenamed('SP.POP.65UP.TO', 'population_age_65_and_above') \\\n",
    "    .withColumnRenamed('SP.POP.TOTL', 'population')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "28ffa050-2883-4f3d-b7d5-b02d456538de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "62"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# You can iterate over a dataframe that is already computed by caching it onces and using it repeatedly\n",
    "yearPivotDF.cache()\n",
    "\n",
    "#Forces the data to be cached\n",
    "yearPivotDF.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "6b0211b6-8397-42c2-9334-4b7f10bbc763",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population_age_0_to_14 < 0').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "26210635-db02-412e-9dc6-87d649f4d7a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population_age_15_64 < 0').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "4d2a051a-0937-43d6-a18a-b8c88c651009",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population_age_65_and_above < 0').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "30f85eac-a7ba-4f89-b63a-c4bc032b2b17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population < 0').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "1405eaef-20e1-4c6b-881b-5d1bb35d81bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('cellular_subscriptions < 0').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "343b139f-5cbc-4087-b456-307c49daa54f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('broadband_subscriptions < 0').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "950c02a3-2c67-4f93-8d98-a22f9a4fe369",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population_age_0_to_14 > population').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "aecb2a20-9693-4e27-9ea9-2aa52c66b3d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population_age_15_64 > population').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "319bf63d-0a95-4aac-93da-b50071d40081",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('population_age_65_and_above > population').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "b8e630cf-9a90-4645-a67b-25e401ef4ec7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearPivotDF.filter('(population_age_0_to_14 + population_age_15_64 + population_age_65_and_above) > population').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "eda3da54-f8a9-4a7a-97db-75e02ab33ebe",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "#Write the yearly totals to a CSV File\n",
    "yearPivotDF \\\n",
    "    .select('year'\n",
    "            , F.col('population').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_0_to_14').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_15_64').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_65_and_above').cast(DecimalType(38, 2))\n",
    "            , F.col('broadband_subscriptions').cast(DecimalType(38, 2))\n",
    "            , F.col('cellular_subscriptions').cast(DecimalType(38, 2))) \\\n",
    "    .coalesce(1) \\\n",
    "    .write.csv('output/YearlyStats', mode='overwrite', header='true')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "356bd4e4",
   "metadata": {},
   "source": [
    "Getting yearly regional totals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "8dad38bd-62b2-4014-8ded-f37e5908404a",
   "metadata": {},
   "outputs": [],
   "source": [
    "regionalIndicators = indicatorsDF.join(countryDimFinal\n",
    "                                       , indicatorsDF.wb_country_code == countryDim.wb_country_code\n",
    "                                       , \"inner\") \\\n",
    "    .select(countryDim.region\n",
    "            , indicatorsDF.wb_country_code\n",
    "            , indicatorsDF.year\n",
    "            , indicatorsDF.indicator_code\n",
    "            , indicatorsDF.indicator_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "7f9fe6f2-eb8f-41e8-8c4c-3efefa42d848",
   "metadata": {},
   "outputs": [],
   "source": [
    "regionalPivot = regionalIndicators.groupBy('region', 'year').pivot('indicator_code').sum('indicator_value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "6e426fec-6e89-4a92-bdcf-0cdda14fbbd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>region</th>\n",
       "      <th>year</th>\n",
       "      <th>IT.CEL.SETS</th>\n",
       "      <th>IT.NET.BBND</th>\n",
       "      <th>SP.POP.0014.TO</th>\n",
       "      <th>SP.POP.1564.TO</th>\n",
       "      <th>SP.POP.65UP.TO</th>\n",
       "      <th>SP.POP.TOTL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>1960</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.095282e+10</td>\n",
       "      <td>1.641773e+10</td>\n",
       "      <td>1.297286e+09</td>\n",
       "      <td>2.868082e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>None</td>\n",
       "      <td>1961</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.115085e+10</td>\n",
       "      <td>1.655578e+10</td>\n",
       "      <td>1.325576e+09</td>\n",
       "      <td>2.904543e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None</td>\n",
       "      <td>1962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.142464e+10</td>\n",
       "      <td>1.676458e+10</td>\n",
       "      <td>1.354457e+09</td>\n",
       "      <td>2.955727e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>None</td>\n",
       "      <td>1963</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.175093e+10</td>\n",
       "      <td>1.705343e+10</td>\n",
       "      <td>1.384791e+09</td>\n",
       "      <td>3.020309e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>None</td>\n",
       "      <td>1964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.205201e+10</td>\n",
       "      <td>1.737373e+10</td>\n",
       "      <td>1.413591e+09</td>\n",
       "      <td>3.085362e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>None</td>\n",
       "      <td>1965</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.232290e+10</td>\n",
       "      <td>1.774128e+10</td>\n",
       "      <td>1.442054e+09</td>\n",
       "      <td>3.152088e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>None</td>\n",
       "      <td>1966</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.263547e+10</td>\n",
       "      <td>1.808430e+10</td>\n",
       "      <td>1.491255e+09</td>\n",
       "      <td>3.222604e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>None</td>\n",
       "      <td>1967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.289914e+10</td>\n",
       "      <td>1.847366e+10</td>\n",
       "      <td>1.540245e+09</td>\n",
       "      <td>3.292842e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>None</td>\n",
       "      <td>1968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.313791e+10</td>\n",
       "      <td>1.890542e+10</td>\n",
       "      <td>1.589418e+09</td>\n",
       "      <td>3.364847e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>None</td>\n",
       "      <td>1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.337932e+10</td>\n",
       "      <td>1.936133e+10</td>\n",
       "      <td>1.638569e+09</td>\n",
       "      <td>3.439530e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>None</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.363070e+10</td>\n",
       "      <td>1.982434e+10</td>\n",
       "      <td>1.687025e+09</td>\n",
       "      <td>3.515850e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>None</td>\n",
       "      <td>1971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.391711e+10</td>\n",
       "      <td>2.026499e+10</td>\n",
       "      <td>1.746787e+09</td>\n",
       "      <td>3.594567e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>None</td>\n",
       "      <td>1972</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.418773e+10</td>\n",
       "      <td>2.070892e+10</td>\n",
       "      <td>1.803971e+09</td>\n",
       "      <td>3.671780e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>None</td>\n",
       "      <td>1973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.444653e+10</td>\n",
       "      <td>2.116428e+10</td>\n",
       "      <td>1.859632e+09</td>\n",
       "      <td>3.748799e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>None</td>\n",
       "      <td>1974</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.468330e+10</td>\n",
       "      <td>2.163665e+10</td>\n",
       "      <td>1.914767e+09</td>\n",
       "      <td>3.825264e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>None</td>\n",
       "      <td>1975</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.488818e+10</td>\n",
       "      <td>2.212653e+10</td>\n",
       "      <td>1.969621e+09</td>\n",
       "      <td>3.900263e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>None</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.509002e+10</td>\n",
       "      <td>2.259919e+10</td>\n",
       "      <td>2.036186e+09</td>\n",
       "      <td>3.974407e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>None</td>\n",
       "      <td>1977</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.526133e+10</td>\n",
       "      <td>2.309704e+10</td>\n",
       "      <td>2.102641e+09</td>\n",
       "      <td>4.048006e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>None</td>\n",
       "      <td>1978</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.541298e+10</td>\n",
       "      <td>2.362847e+10</td>\n",
       "      <td>2.167806e+09</td>\n",
       "      <td>4.122868e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>None</td>\n",
       "      <td>1979</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.555506e+10</td>\n",
       "      <td>2.419199e+10</td>\n",
       "      <td>2.228448e+09</td>\n",
       "      <td>4.199531e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>None</td>\n",
       "      <td>1980</td>\n",
       "      <td>1.643740e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.569147e+10</td>\n",
       "      <td>2.477711e+10</td>\n",
       "      <td>2.282277e+09</td>\n",
       "      <td>4.277106e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>None</td>\n",
       "      <td>1981</td>\n",
       "      <td>3.948430e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.583730e+10</td>\n",
       "      <td>2.536994e+10</td>\n",
       "      <td>2.340015e+09</td>\n",
       "      <td>4.356785e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>None</td>\n",
       "      <td>1982</td>\n",
       "      <td>6.097410e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.599798e+10</td>\n",
       "      <td>2.599022e+10</td>\n",
       "      <td>2.390013e+09</td>\n",
       "      <td>4.439919e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>None</td>\n",
       "      <td>1983</td>\n",
       "      <td>8.811210e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.616144e+10</td>\n",
       "      <td>2.662266e+10</td>\n",
       "      <td>2.435238e+09</td>\n",
       "      <td>4.524071e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>None</td>\n",
       "      <td>1984</td>\n",
       "      <td>1.796544e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.632075e+10</td>\n",
       "      <td>2.725807e+10</td>\n",
       "      <td>2.480610e+09</td>\n",
       "      <td>4.608123e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>None</td>\n",
       "      <td>1985</td>\n",
       "      <td>4.058062e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.648397e+10</td>\n",
       "      <td>2.790251e+10</td>\n",
       "      <td>2.530182e+09</td>\n",
       "      <td>4.693891e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>None</td>\n",
       "      <td>1986</td>\n",
       "      <td>7.838225e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.671258e+10</td>\n",
       "      <td>2.849412e+10</td>\n",
       "      <td>2.593515e+09</td>\n",
       "      <td>4.782293e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>None</td>\n",
       "      <td>1987</td>\n",
       "      <td>1.371336e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.694276e+10</td>\n",
       "      <td>2.910681e+10</td>\n",
       "      <td>2.659224e+09</td>\n",
       "      <td>4.873199e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>None</td>\n",
       "      <td>1988</td>\n",
       "      <td>2.327014e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.717145e+10</td>\n",
       "      <td>2.972906e+10</td>\n",
       "      <td>2.727352e+09</td>\n",
       "      <td>4.965156e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>None</td>\n",
       "      <td>1989</td>\n",
       "      <td>3.938226e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.739079e+10</td>\n",
       "      <td>3.035648e+10</td>\n",
       "      <td>2.798376e+09</td>\n",
       "      <td>5.056982e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>None</td>\n",
       "      <td>1990</td>\n",
       "      <td>6.085150e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.760131e+10</td>\n",
       "      <td>3.100020e+10</td>\n",
       "      <td>2.873165e+09</td>\n",
       "      <td>5.149931e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>None</td>\n",
       "      <td>1991</td>\n",
       "      <td>8.966970e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.785208e+10</td>\n",
       "      <td>3.157222e+10</td>\n",
       "      <td>2.958423e+09</td>\n",
       "      <td>5.240782e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>None</td>\n",
       "      <td>1992</td>\n",
       "      <td>1.299660e+08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.804149e+10</td>\n",
       "      <td>3.217666e+10</td>\n",
       "      <td>3.047427e+09</td>\n",
       "      <td>5.329109e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>None</td>\n",
       "      <td>1993</td>\n",
       "      <td>1.953725e+08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.819627e+10</td>\n",
       "      <td>3.281465e+10</td>\n",
       "      <td>3.138088e+09</td>\n",
       "      <td>5.417492e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>None</td>\n",
       "      <td>1994</td>\n",
       "      <td>3.254472e+08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.833929e+10</td>\n",
       "      <td>3.345839e+10</td>\n",
       "      <td>3.225844e+09</td>\n",
       "      <td>5.504983e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>None</td>\n",
       "      <td>1995</td>\n",
       "      <td>5.450387e+08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.848343e+10</td>\n",
       "      <td>3.410310e+10</td>\n",
       "      <td>3.309014e+09</td>\n",
       "      <td>5.592223e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>None</td>\n",
       "      <td>1996</td>\n",
       "      <td>8.885198e+08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.856564e+10</td>\n",
       "      <td>3.478324e+10</td>\n",
       "      <td>3.407969e+09</td>\n",
       "      <td>5.678392e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>None</td>\n",
       "      <td>1997</td>\n",
       "      <td>1.372889e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.868362e+10</td>\n",
       "      <td>3.543514e+10</td>\n",
       "      <td>3.497698e+09</td>\n",
       "      <td>5.764385e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>None</td>\n",
       "      <td>1998</td>\n",
       "      <td>2.088890e+09</td>\n",
       "      <td>8.459000e+05</td>\n",
       "      <td>1.880463e+10</td>\n",
       "      <td>3.608207e+10</td>\n",
       "      <td>3.582343e+09</td>\n",
       "      <td>5.849515e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>None</td>\n",
       "      <td>1999</td>\n",
       "      <td>3.346237e+09</td>\n",
       "      <td>3.336286e+06</td>\n",
       "      <td>1.888338e+10</td>\n",
       "      <td>3.675991e+10</td>\n",
       "      <td>3.667869e+09</td>\n",
       "      <td>5.933506e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>None</td>\n",
       "      <td>2000</td>\n",
       "      <td>5.281324e+09</td>\n",
       "      <td>6.392553e+07</td>\n",
       "      <td>1.890479e+10</td>\n",
       "      <td>3.748348e+10</td>\n",
       "      <td>3.757238e+09</td>\n",
       "      <td>6.016878e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>None</td>\n",
       "      <td>2001</td>\n",
       "      <td>7.211755e+09</td>\n",
       "      <td>1.909957e+08</td>\n",
       "      <td>1.896296e+10</td>\n",
       "      <td>3.815660e+10</td>\n",
       "      <td>3.856337e+09</td>\n",
       "      <td>6.099923e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>None</td>\n",
       "      <td>2002</td>\n",
       "      <td>9.079581e+09</td>\n",
       "      <td>3.751730e+08</td>\n",
       "      <td>1.896104e+10</td>\n",
       "      <td>3.888615e+10</td>\n",
       "      <td>3.958560e+09</td>\n",
       "      <td>6.182915e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>None</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.139913e+10</td>\n",
       "      <td>6.338633e+08</td>\n",
       "      <td>1.892601e+10</td>\n",
       "      <td>3.965253e+10</td>\n",
       "      <td>4.060616e+09</td>\n",
       "      <td>6.266263e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>None</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.469758e+10</td>\n",
       "      <td>1.039062e+09</td>\n",
       "      <td>1.890461e+10</td>\n",
       "      <td>4.041768e+10</td>\n",
       "      <td>4.155481e+09</td>\n",
       "      <td>6.350132e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>None</td>\n",
       "      <td>2005</td>\n",
       "      <td>1.900144e+10</td>\n",
       "      <td>1.490714e+09</td>\n",
       "      <td>1.892173e+10</td>\n",
       "      <td>4.116044e+10</td>\n",
       "      <td>4.240414e+09</td>\n",
       "      <td>6.434619e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>None</td>\n",
       "      <td>2006</td>\n",
       "      <td>2.426191e+10</td>\n",
       "      <td>1.991198e+09</td>\n",
       "      <td>1.897751e+10</td>\n",
       "      <td>4.185942e+10</td>\n",
       "      <td>4.336222e+09</td>\n",
       "      <td>6.519694e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>None</td>\n",
       "      <td>2007</td>\n",
       "      <td>3.039204e+10</td>\n",
       "      <td>2.555962e+09</td>\n",
       "      <td>1.906525e+10</td>\n",
       "      <td>4.253994e+10</td>\n",
       "      <td>4.423685e+09</td>\n",
       "      <td>6.605284e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>None</td>\n",
       "      <td>2008</td>\n",
       "      <td>3.709154e+10</td>\n",
       "      <td>3.095569e+09</td>\n",
       "      <td>1.918164e+10</td>\n",
       "      <td>4.320715e+10</td>\n",
       "      <td>4.508957e+09</td>\n",
       "      <td>6.692189e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>None</td>\n",
       "      <td>2009</td>\n",
       "      <td>4.317485e+10</td>\n",
       "      <td>3.643811e+09</td>\n",
       "      <td>1.930785e+10</td>\n",
       "      <td>4.386429e+10</td>\n",
       "      <td>4.598834e+09</td>\n",
       "      <td>6.779529e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>None</td>\n",
       "      <td>2010</td>\n",
       "      <td>4.984186e+10</td>\n",
       "      <td>4.171374e+09</td>\n",
       "      <td>1.943423e+10</td>\n",
       "      <td>4.451555e+10</td>\n",
       "      <td>4.698322e+09</td>\n",
       "      <td>6.867259e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>None</td>\n",
       "      <td>2011</td>\n",
       "      <td>5.592462e+10</td>\n",
       "      <td>4.802385e+09</td>\n",
       "      <td>1.958343e+10</td>\n",
       "      <td>4.513191e+10</td>\n",
       "      <td>4.818474e+09</td>\n",
       "      <td>6.955848e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>None</td>\n",
       "      <td>2012</td>\n",
       "      <td>5.998158e+10</td>\n",
       "      <td>5.279837e+09</td>\n",
       "      <td>1.973513e+10</td>\n",
       "      <td>4.577204e+10</td>\n",
       "      <td>4.949044e+09</td>\n",
       "      <td>7.048107e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>None</td>\n",
       "      <td>2013</td>\n",
       "      <td>6.430747e+10</td>\n",
       "      <td>5.664630e+09</td>\n",
       "      <td>1.988701e+10</td>\n",
       "      <td>4.641150e+10</td>\n",
       "      <td>5.091297e+09</td>\n",
       "      <td>7.141480e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>None</td>\n",
       "      <td>2014</td>\n",
       "      <td>6.772183e+10</td>\n",
       "      <td>6.011573e+09</td>\n",
       "      <td>2.004045e+10</td>\n",
       "      <td>4.703609e+10</td>\n",
       "      <td>5.249877e+09</td>\n",
       "      <td>7.235138e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>None</td>\n",
       "      <td>2015</td>\n",
       "      <td>6.962964e+10</td>\n",
       "      <td>7.150559e+09</td>\n",
       "      <td>2.019438e+10</td>\n",
       "      <td>4.763901e+10</td>\n",
       "      <td>5.427879e+09</td>\n",
       "      <td>7.328597e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>None</td>\n",
       "      <td>2016</td>\n",
       "      <td>7.280884e+10</td>\n",
       "      <td>7.920022e+09</td>\n",
       "      <td>2.035680e+10</td>\n",
       "      <td>4.820547e+10</td>\n",
       "      <td>5.637846e+09</td>\n",
       "      <td>7.422473e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>None</td>\n",
       "      <td>2017</td>\n",
       "      <td>7.536907e+10</td>\n",
       "      <td>9.044838e+09</td>\n",
       "      <td>2.051620e+10</td>\n",
       "      <td>4.876196e+10</td>\n",
       "      <td>5.864928e+09</td>\n",
       "      <td>7.516787e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>None</td>\n",
       "      <td>2018</td>\n",
       "      <td>7.674181e+10</td>\n",
       "      <td>9.451505e+09</td>\n",
       "      <td>2.066564e+10</td>\n",
       "      <td>4.929878e+10</td>\n",
       "      <td>6.100857e+09</td>\n",
       "      <td>7.609015e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>None</td>\n",
       "      <td>2019</td>\n",
       "      <td>8.084976e+10</td>\n",
       "      <td>1.019242e+10</td>\n",
       "      <td>2.080142e+10</td>\n",
       "      <td>4.983010e+10</td>\n",
       "      <td>6.336214e+09</td>\n",
       "      <td>7.699255e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>None</td>\n",
       "      <td>2020</td>\n",
       "      <td>8.156693e+10</td>\n",
       "      <td>1.104373e+10</td>\n",
       "      <td>2.092159e+10</td>\n",
       "      <td>5.036636e+10</td>\n",
       "      <td>6.567539e+09</td>\n",
       "      <td>7.788034e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>None</td>\n",
       "      <td>2021</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.103378e+10</td>\n",
       "      <td>5.086278e+10</td>\n",
       "      <td>6.794933e+09</td>\n",
       "      <td>7.871655e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1960</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.307162e+08</td>\n",
       "      <td>1.967120e+08</td>\n",
       "      <td>1.467246e+07</td>\n",
       "      <td>3.421649e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1961</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.342636e+08</td>\n",
       "      <td>2.006306e+08</td>\n",
       "      <td>1.508250e+07</td>\n",
       "      <td>3.500427e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.375952e+08</td>\n",
       "      <td>2.050370e+08</td>\n",
       "      <td>1.549967e+07</td>\n",
       "      <td>3.581997e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1963</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.407616e+08</td>\n",
       "      <td>2.097619e+08</td>\n",
       "      <td>1.591555e+07</td>\n",
       "      <td>3.665090e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.439339e+08</td>\n",
       "      <td>2.146404e+08</td>\n",
       "      <td>1.633057e+07</td>\n",
       "      <td>3.749770e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1965</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.472250e+08</td>\n",
       "      <td>2.196196e+08</td>\n",
       "      <td>1.674751e+07</td>\n",
       "      <td>3.836663e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1966</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.506428e+08</td>\n",
       "      <td>2.243589e+08</td>\n",
       "      <td>1.729126e+07</td>\n",
       "      <td>3.923692e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.541530e+08</td>\n",
       "      <td>2.291994e+08</td>\n",
       "      <td>1.782376e+07</td>\n",
       "      <td>4.012545e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.577051e+08</td>\n",
       "      <td>2.342606e+08</td>\n",
       "      <td>1.836243e+07</td>\n",
       "      <td>4.104086e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.611620e+08</td>\n",
       "      <td>2.396281e+08</td>\n",
       "      <td>1.891442e+07</td>\n",
       "      <td>4.197872e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.644432e+08</td>\n",
       "      <td>2.452817e+08</td>\n",
       "      <td>1.947763e+07</td>\n",
       "      <td>4.292873e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.681233e+08</td>\n",
       "      <td>2.517074e+08</td>\n",
       "      <td>2.023530e+07</td>\n",
       "      <td>4.401532e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1972</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.713602e+08</td>\n",
       "      <td>2.577571e+08</td>\n",
       "      <td>2.090794e+07</td>\n",
       "      <td>4.501150e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.743883e+08</td>\n",
       "      <td>2.640788e+08</td>\n",
       "      <td>2.157466e+07</td>\n",
       "      <td>4.601340e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1974</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.771748e+08</td>\n",
       "      <td>2.706780e+08</td>\n",
       "      <td>2.225735e+07</td>\n",
       "      <td>4.702047e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1975</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.795566e+08</td>\n",
       "      <td>2.773203e+08</td>\n",
       "      <td>2.293796e+07</td>\n",
       "      <td>4.799116e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.819529e+08</td>\n",
       "      <td>2.833237e+08</td>\n",
       "      <td>2.379817e+07</td>\n",
       "      <td>4.891732e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1977</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.837983e+08</td>\n",
       "      <td>2.896626e+08</td>\n",
       "      <td>2.468552e+07</td>\n",
       "      <td>4.982465e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1978</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.852872e+08</td>\n",
       "      <td>2.963171e+08</td>\n",
       "      <td>2.558098e+07</td>\n",
       "      <td>5.072871e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1979</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.867423e+08</td>\n",
       "      <td>3.032990e+08</td>\n",
       "      <td>2.645125e+07</td>\n",
       "      <td>5.165966e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1980</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.882681e+08</td>\n",
       "      <td>3.104153e+08</td>\n",
       "      <td>2.726565e+07</td>\n",
       "      <td>5.260565e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1981</td>\n",
       "      <td>1.327500e+04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.894369e+08</td>\n",
       "      <td>3.181675e+08</td>\n",
       "      <td>2.819578e+07</td>\n",
       "      <td>5.359120e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1982</td>\n",
       "      <td>1.980400e+04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.909421e+08</td>\n",
       "      <td>3.258823e+08</td>\n",
       "      <td>2.904980e+07</td>\n",
       "      <td>5.459911e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1983</td>\n",
       "      <td>2.719800e+04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.925130e+08</td>\n",
       "      <td>3.336315e+08</td>\n",
       "      <td>2.986612e+07</td>\n",
       "      <td>5.561335e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1984</td>\n",
       "      <td>4.314200e+04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.937511e+08</td>\n",
       "      <td>3.416132e+08</td>\n",
       "      <td>3.070647e+07</td>\n",
       "      <td>5.662003e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1985</td>\n",
       "      <td>6.822900e+04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.944877e+08</td>\n",
       "      <td>3.499271e+08</td>\n",
       "      <td>3.161810e+07</td>\n",
       "      <td>5.761697e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1986</td>\n",
       "      <td>1.283900e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.955120e+08</td>\n",
       "      <td>3.577778e+08</td>\n",
       "      <td>3.264572e+07</td>\n",
       "      <td>5.860801e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1987</td>\n",
       "      <td>2.255250e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.958831e+08</td>\n",
       "      <td>3.662144e+08</td>\n",
       "      <td>3.374322e+07</td>\n",
       "      <td>5.959935e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1988</td>\n",
       "      <td>4.208430e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.958469e+08</td>\n",
       "      <td>3.749277e+08</td>\n",
       "      <td>3.490423e+07</td>\n",
       "      <td>6.058397e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1989</td>\n",
       "      <td>8.618320e+05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.958646e+08</td>\n",
       "      <td>3.835195e+08</td>\n",
       "      <td>3.612329e+07</td>\n",
       "      <td>6.156755e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1990</td>\n",
       "      <td>1.544709e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.961339e+08</td>\n",
       "      <td>3.916441e+08</td>\n",
       "      <td>3.738352e+07</td>\n",
       "      <td>6.253354e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1991</td>\n",
       "      <td>2.500963e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.965301e+08</td>\n",
       "      <td>3.995683e+08</td>\n",
       "      <td>3.881522e+07</td>\n",
       "      <td>6.350916e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1992</td>\n",
       "      <td>3.495450e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.972465e+08</td>\n",
       "      <td>4.069391e+08</td>\n",
       "      <td>4.025348e+07</td>\n",
       "      <td>6.446197e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1993</td>\n",
       "      <td>4.852549e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.980904e+08</td>\n",
       "      <td>4.139955e+08</td>\n",
       "      <td>4.171838e+07</td>\n",
       "      <td>6.539867e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1994</td>\n",
       "      <td>9.098743e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.986857e+08</td>\n",
       "      <td>4.211709e+08</td>\n",
       "      <td>4.324033e+07</td>\n",
       "      <td>6.632810e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1995</td>\n",
       "      <td>2.019611e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.988123e+08</td>\n",
       "      <td>4.286210e+08</td>\n",
       "      <td>4.482990e+07</td>\n",
       "      <td>6.724498e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1996</td>\n",
       "      <td>4.145429e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.990416e+08</td>\n",
       "      <td>4.359237e+08</td>\n",
       "      <td>4.656504e+07</td>\n",
       "      <td>6.817202e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>East Asia &amp; Pacific</td>\n",
       "      <td>1997</td>\n",
       "      <td>6.014753e+07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.986616e+08</td>\n",
       "      <td>4.434361e+08</td>\n",
       "      <td>4.832169e+07</td>\n",
       "      <td>6.906132e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 region  year   IT.CEL.SETS   IT.NET.BBND  SP.POP.0014.TO  \\\n",
       "0                  None  1960  0.000000e+00           NaN    1.095282e+10   \n",
       "1                  None  1961           NaN           NaN    1.115085e+10   \n",
       "2                  None  1962           NaN           NaN    1.142464e+10   \n",
       "3                  None  1963           NaN           NaN    1.175093e+10   \n",
       "4                  None  1964           NaN           NaN    1.205201e+10   \n",
       "5                  None  1965  0.000000e+00           NaN    1.232290e+10   \n",
       "6                  None  1966           NaN           NaN    1.263547e+10   \n",
       "7                  None  1967           NaN           NaN    1.289914e+10   \n",
       "8                  None  1968           NaN           NaN    1.313791e+10   \n",
       "9                  None  1969           NaN           NaN    1.337932e+10   \n",
       "10                 None  1970  0.000000e+00           NaN    1.363070e+10   \n",
       "11                 None  1971           NaN           NaN    1.391711e+10   \n",
       "12                 None  1972           NaN           NaN    1.418773e+10   \n",
       "13                 None  1973           NaN           NaN    1.444653e+10   \n",
       "14                 None  1974           NaN           NaN    1.468330e+10   \n",
       "15                 None  1975  0.000000e+00           NaN    1.488818e+10   \n",
       "16                 None  1976  0.000000e+00           NaN    1.509002e+10   \n",
       "17                 None  1977  0.000000e+00           NaN    1.526133e+10   \n",
       "18                 None  1978  0.000000e+00           NaN    1.541298e+10   \n",
       "19                 None  1979  0.000000e+00           NaN    1.555506e+10   \n",
       "20                 None  1980  1.643740e+05           NaN    1.569147e+10   \n",
       "21                 None  1981  3.948430e+05           NaN    1.583730e+10   \n",
       "22                 None  1982  6.097410e+05           NaN    1.599798e+10   \n",
       "23                 None  1983  8.811210e+05           NaN    1.616144e+10   \n",
       "24                 None  1984  1.796544e+06           NaN    1.632075e+10   \n",
       "25                 None  1985  4.058062e+06           NaN    1.648397e+10   \n",
       "26                 None  1986  7.838225e+06           NaN    1.671258e+10   \n",
       "27                 None  1987  1.371336e+07           NaN    1.694276e+10   \n",
       "28                 None  1988  2.327014e+07           NaN    1.717145e+10   \n",
       "29                 None  1989  3.938226e+07           NaN    1.739079e+10   \n",
       "30                 None  1990  6.085150e+07           NaN    1.760131e+10   \n",
       "31                 None  1991  8.966970e+07           NaN    1.785208e+10   \n",
       "32                 None  1992  1.299660e+08           NaN    1.804149e+10   \n",
       "33                 None  1993  1.953725e+08           NaN    1.819627e+10   \n",
       "34                 None  1994  3.254472e+08           NaN    1.833929e+10   \n",
       "35                 None  1995  5.450387e+08           NaN    1.848343e+10   \n",
       "36                 None  1996  8.885198e+08           NaN    1.856564e+10   \n",
       "37                 None  1997  1.372889e+09           NaN    1.868362e+10   \n",
       "38                 None  1998  2.088890e+09  8.459000e+05    1.880463e+10   \n",
       "39                 None  1999  3.346237e+09  3.336286e+06    1.888338e+10   \n",
       "40                 None  2000  5.281324e+09  6.392553e+07    1.890479e+10   \n",
       "41                 None  2001  7.211755e+09  1.909957e+08    1.896296e+10   \n",
       "42                 None  2002  9.079581e+09  3.751730e+08    1.896104e+10   \n",
       "43                 None  2003  1.139913e+10  6.338633e+08    1.892601e+10   \n",
       "44                 None  2004  1.469758e+10  1.039062e+09    1.890461e+10   \n",
       "45                 None  2005  1.900144e+10  1.490714e+09    1.892173e+10   \n",
       "46                 None  2006  2.426191e+10  1.991198e+09    1.897751e+10   \n",
       "47                 None  2007  3.039204e+10  2.555962e+09    1.906525e+10   \n",
       "48                 None  2008  3.709154e+10  3.095569e+09    1.918164e+10   \n",
       "49                 None  2009  4.317485e+10  3.643811e+09    1.930785e+10   \n",
       "50                 None  2010  4.984186e+10  4.171374e+09    1.943423e+10   \n",
       "51                 None  2011  5.592462e+10  4.802385e+09    1.958343e+10   \n",
       "52                 None  2012  5.998158e+10  5.279837e+09    1.973513e+10   \n",
       "53                 None  2013  6.430747e+10  5.664630e+09    1.988701e+10   \n",
       "54                 None  2014  6.772183e+10  6.011573e+09    2.004045e+10   \n",
       "55                 None  2015  6.962964e+10  7.150559e+09    2.019438e+10   \n",
       "56                 None  2016  7.280884e+10  7.920022e+09    2.035680e+10   \n",
       "57                 None  2017  7.536907e+10  9.044838e+09    2.051620e+10   \n",
       "58                 None  2018  7.674181e+10  9.451505e+09    2.066564e+10   \n",
       "59                 None  2019  8.084976e+10  1.019242e+10    2.080142e+10   \n",
       "60                 None  2020  8.156693e+10  1.104373e+10    2.092159e+10   \n",
       "61                 None  2021           NaN           NaN    2.103378e+10   \n",
       "62  East Asia & Pacific  1960  0.000000e+00           NaN    1.307162e+08   \n",
       "63  East Asia & Pacific  1961           NaN           NaN    1.342636e+08   \n",
       "64  East Asia & Pacific  1962           NaN           NaN    1.375952e+08   \n",
       "65  East Asia & Pacific  1963           NaN           NaN    1.407616e+08   \n",
       "66  East Asia & Pacific  1964           NaN           NaN    1.439339e+08   \n",
       "67  East Asia & Pacific  1965  0.000000e+00           NaN    1.472250e+08   \n",
       "68  East Asia & Pacific  1966           NaN           NaN    1.506428e+08   \n",
       "69  East Asia & Pacific  1967           NaN           NaN    1.541530e+08   \n",
       "70  East Asia & Pacific  1968           NaN           NaN    1.577051e+08   \n",
       "71  East Asia & Pacific  1969           NaN           NaN    1.611620e+08   \n",
       "72  East Asia & Pacific  1970  0.000000e+00           NaN    1.644432e+08   \n",
       "73  East Asia & Pacific  1971           NaN           NaN    1.681233e+08   \n",
       "74  East Asia & Pacific  1972           NaN           NaN    1.713602e+08   \n",
       "75  East Asia & Pacific  1973           NaN           NaN    1.743883e+08   \n",
       "76  East Asia & Pacific  1974           NaN           NaN    1.771748e+08   \n",
       "77  East Asia & Pacific  1975  0.000000e+00           NaN    1.795566e+08   \n",
       "78  East Asia & Pacific  1976  0.000000e+00           NaN    1.819529e+08   \n",
       "79  East Asia & Pacific  1977  0.000000e+00           NaN    1.837983e+08   \n",
       "80  East Asia & Pacific  1978  0.000000e+00           NaN    1.852872e+08   \n",
       "81  East Asia & Pacific  1979  0.000000e+00           NaN    1.867423e+08   \n",
       "82  East Asia & Pacific  1980  0.000000e+00           NaN    1.882681e+08   \n",
       "83  East Asia & Pacific  1981  1.327500e+04           NaN    1.894369e+08   \n",
       "84  East Asia & Pacific  1982  1.980400e+04           NaN    1.909421e+08   \n",
       "85  East Asia & Pacific  1983  2.719800e+04           NaN    1.925130e+08   \n",
       "86  East Asia & Pacific  1984  4.314200e+04           NaN    1.937511e+08   \n",
       "87  East Asia & Pacific  1985  6.822900e+04           NaN    1.944877e+08   \n",
       "88  East Asia & Pacific  1986  1.283900e+05           NaN    1.955120e+08   \n",
       "89  East Asia & Pacific  1987  2.255250e+05           NaN    1.958831e+08   \n",
       "90  East Asia & Pacific  1988  4.208430e+05           NaN    1.958469e+08   \n",
       "91  East Asia & Pacific  1989  8.618320e+05           NaN    1.958646e+08   \n",
       "92  East Asia & Pacific  1990  1.544709e+06           NaN    1.961339e+08   \n",
       "93  East Asia & Pacific  1991  2.500963e+06           NaN    1.965301e+08   \n",
       "94  East Asia & Pacific  1992  3.495450e+06           NaN    1.972465e+08   \n",
       "95  East Asia & Pacific  1993  4.852549e+06           NaN    1.980904e+08   \n",
       "96  East Asia & Pacific  1994  9.098743e+06           NaN    1.986857e+08   \n",
       "97  East Asia & Pacific  1995  2.019611e+07           NaN    1.988123e+08   \n",
       "98  East Asia & Pacific  1996  4.145429e+07           NaN    1.990416e+08   \n",
       "99  East Asia & Pacific  1997  6.014753e+07           NaN    1.986616e+08   \n",
       "\n",
       "    SP.POP.1564.TO  SP.POP.65UP.TO   SP.POP.TOTL  \n",
       "0     1.641773e+10    1.297286e+09  2.868082e+10  \n",
       "1     1.655578e+10    1.325576e+09  2.904543e+10  \n",
       "2     1.676458e+10    1.354457e+09  2.955727e+10  \n",
       "3     1.705343e+10    1.384791e+09  3.020309e+10  \n",
       "4     1.737373e+10    1.413591e+09  3.085362e+10  \n",
       "5     1.774128e+10    1.442054e+09  3.152088e+10  \n",
       "6     1.808430e+10    1.491255e+09  3.222604e+10  \n",
       "7     1.847366e+10    1.540245e+09  3.292842e+10  \n",
       "8     1.890542e+10    1.589418e+09  3.364847e+10  \n",
       "9     1.936133e+10    1.638569e+09  3.439530e+10  \n",
       "10    1.982434e+10    1.687025e+09  3.515850e+10  \n",
       "11    2.026499e+10    1.746787e+09  3.594567e+10  \n",
       "12    2.070892e+10    1.803971e+09  3.671780e+10  \n",
       "13    2.116428e+10    1.859632e+09  3.748799e+10  \n",
       "14    2.163665e+10    1.914767e+09  3.825264e+10  \n",
       "15    2.212653e+10    1.969621e+09  3.900263e+10  \n",
       "16    2.259919e+10    2.036186e+09  3.974407e+10  \n",
       "17    2.309704e+10    2.102641e+09  4.048006e+10  \n",
       "18    2.362847e+10    2.167806e+09  4.122868e+10  \n",
       "19    2.419199e+10    2.228448e+09  4.199531e+10  \n",
       "20    2.477711e+10    2.282277e+09  4.277106e+10  \n",
       "21    2.536994e+10    2.340015e+09  4.356785e+10  \n",
       "22    2.599022e+10    2.390013e+09  4.439919e+10  \n",
       "23    2.662266e+10    2.435238e+09  4.524071e+10  \n",
       "24    2.725807e+10    2.480610e+09  4.608123e+10  \n",
       "25    2.790251e+10    2.530182e+09  4.693891e+10  \n",
       "26    2.849412e+10    2.593515e+09  4.782293e+10  \n",
       "27    2.910681e+10    2.659224e+09  4.873199e+10  \n",
       "28    2.972906e+10    2.727352e+09  4.965156e+10  \n",
       "29    3.035648e+10    2.798376e+09  5.056982e+10  \n",
       "30    3.100020e+10    2.873165e+09  5.149931e+10  \n",
       "31    3.157222e+10    2.958423e+09  5.240782e+10  \n",
       "32    3.217666e+10    3.047427e+09  5.329109e+10  \n",
       "33    3.281465e+10    3.138088e+09  5.417492e+10  \n",
       "34    3.345839e+10    3.225844e+09  5.504983e+10  \n",
       "35    3.410310e+10    3.309014e+09  5.592223e+10  \n",
       "36    3.478324e+10    3.407969e+09  5.678392e+10  \n",
       "37    3.543514e+10    3.497698e+09  5.764385e+10  \n",
       "38    3.608207e+10    3.582343e+09  5.849515e+10  \n",
       "39    3.675991e+10    3.667869e+09  5.933506e+10  \n",
       "40    3.748348e+10    3.757238e+09  6.016878e+10  \n",
       "41    3.815660e+10    3.856337e+09  6.099923e+10  \n",
       "42    3.888615e+10    3.958560e+09  6.182915e+10  \n",
       "43    3.965253e+10    4.060616e+09  6.266263e+10  \n",
       "44    4.041768e+10    4.155481e+09  6.350132e+10  \n",
       "45    4.116044e+10    4.240414e+09  6.434619e+10  \n",
       "46    4.185942e+10    4.336222e+09  6.519694e+10  \n",
       "47    4.253994e+10    4.423685e+09  6.605284e+10  \n",
       "48    4.320715e+10    4.508957e+09  6.692189e+10  \n",
       "49    4.386429e+10    4.598834e+09  6.779529e+10  \n",
       "50    4.451555e+10    4.698322e+09  6.867259e+10  \n",
       "51    4.513191e+10    4.818474e+09  6.955848e+10  \n",
       "52    4.577204e+10    4.949044e+09  7.048107e+10  \n",
       "53    4.641150e+10    5.091297e+09  7.141480e+10  \n",
       "54    4.703609e+10    5.249877e+09  7.235138e+10  \n",
       "55    4.763901e+10    5.427879e+09  7.328597e+10  \n",
       "56    4.820547e+10    5.637846e+09  7.422473e+10  \n",
       "57    4.876196e+10    5.864928e+09  7.516787e+10  \n",
       "58    4.929878e+10    6.100857e+09  7.609015e+10  \n",
       "59    4.983010e+10    6.336214e+09  7.699255e+10  \n",
       "60    5.036636e+10    6.567539e+09  7.788034e+10  \n",
       "61    5.086278e+10    6.794933e+09  7.871655e+10  \n",
       "62    1.967120e+08    1.467246e+07  3.421649e+08  \n",
       "63    2.006306e+08    1.508250e+07  3.500427e+08  \n",
       "64    2.050370e+08    1.549967e+07  3.581997e+08  \n",
       "65    2.097619e+08    1.591555e+07  3.665090e+08  \n",
       "66    2.146404e+08    1.633057e+07  3.749770e+08  \n",
       "67    2.196196e+08    1.674751e+07  3.836663e+08  \n",
       "68    2.243589e+08    1.729126e+07  3.923692e+08  \n",
       "69    2.291994e+08    1.782376e+07  4.012545e+08  \n",
       "70    2.342606e+08    1.836243e+07  4.104086e+08  \n",
       "71    2.396281e+08    1.891442e+07  4.197872e+08  \n",
       "72    2.452817e+08    1.947763e+07  4.292873e+08  \n",
       "73    2.517074e+08    2.023530e+07  4.401532e+08  \n",
       "74    2.577571e+08    2.090794e+07  4.501150e+08  \n",
       "75    2.640788e+08    2.157466e+07  4.601340e+08  \n",
       "76    2.706780e+08    2.225735e+07  4.702047e+08  \n",
       "77    2.773203e+08    2.293796e+07  4.799116e+08  \n",
       "78    2.833237e+08    2.379817e+07  4.891732e+08  \n",
       "79    2.896626e+08    2.468552e+07  4.982465e+08  \n",
       "80    2.963171e+08    2.558098e+07  5.072871e+08  \n",
       "81    3.032990e+08    2.645125e+07  5.165966e+08  \n",
       "82    3.104153e+08    2.726565e+07  5.260565e+08  \n",
       "83    3.181675e+08    2.819578e+07  5.359120e+08  \n",
       "84    3.258823e+08    2.904980e+07  5.459911e+08  \n",
       "85    3.336315e+08    2.986612e+07  5.561335e+08  \n",
       "86    3.416132e+08    3.070647e+07  5.662003e+08  \n",
       "87    3.499271e+08    3.161810e+07  5.761697e+08  \n",
       "88    3.577778e+08    3.264572e+07  5.860801e+08  \n",
       "89    3.662144e+08    3.374322e+07  5.959935e+08  \n",
       "90    3.749277e+08    3.490423e+07  6.058397e+08  \n",
       "91    3.835195e+08    3.612329e+07  6.156755e+08  \n",
       "92    3.916441e+08    3.738352e+07  6.253354e+08  \n",
       "93    3.995683e+08    3.881522e+07  6.350916e+08  \n",
       "94    4.069391e+08    4.025348e+07  6.446197e+08  \n",
       "95    4.139955e+08    4.171838e+07  6.539867e+08  \n",
       "96    4.211709e+08    4.324033e+07  6.632810e+08  \n",
       "97    4.286210e+08    4.482990e+07  6.724498e+08  \n",
       "98    4.359237e+08    4.656504e+07  6.817202e+08  \n",
       "99    4.434361e+08    4.832169e+07  6.906132e+08  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showDF(regionalPivot.orderBy('region', 'year'), limitRows=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "e63d8c8e-93e8-44c1-9fde-d4e747a396ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#Write the regional-yearly totals to a CSV File\n",
    "regionalPivot.filter('region is not null') \\\n",
    "    .orderBy('region','year') \\\n",
    "    .withColumnRenamed('IT.CEL.SETS', 'cellular_subscriptions') \\\n",
    "    .withColumnRenamed('IT.NET.BBND', 'broadband_subscriptions') \\\n",
    "    .withColumnRenamed('SP.POP.0014.TO', 'population_age_0_to_14') \\\n",
    "    .withColumnRenamed('SP.POP.1564.TO', 'population_age_15_64') \\\n",
    "    .withColumnRenamed('SP.POP.65UP.TO', 'population_age_65_and_above') \\\n",
    "    .withColumnRenamed('SP.POP.TOTL', 'population') \\\n",
    "    .select('region'\n",
    "            , F.col('year')\n",
    "            , F.col('population').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_0_to_14').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_15_64').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_65_and_above').cast(DecimalType(38, 2))\n",
    "            , F.col('broadband_subscriptions').cast(DecimalType(38, 2))\n",
    "            , F.col('cellular_subscriptions').cast(DecimalType(38, 2))) \\\n",
    "    .coalesce(1) \\\n",
    "    .write.csv('output/RegionalStats', mode='overwrite', header='true')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "198423bd",
   "metadata": {},
   "source": [
    "Becky finds the regional metrics interesting, but she wants to look at these metrics at a country level for each year. Can you adapt the regional pivot that we computed earlier to get the metrics for each year and country ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "89dc3f46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1960</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1961</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1962</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1963</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1964</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1965</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1966</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1967</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1968</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1969</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1970</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1971</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1972</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1973</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1974</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1975</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1976</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1977</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1978</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>AFE</td>\n",
       "      <td>ZH</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>1979</td>\n",
       "      <td>IT.NET.BBND</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code country_iso_code                         name  year  \\\n",
       "0              AFE               ZH  Africa Eastern and Southern  1960   \n",
       "1              AFE               ZH  Africa Eastern and Southern  1961   \n",
       "2              AFE               ZH  Africa Eastern and Southern  1962   \n",
       "3              AFE               ZH  Africa Eastern and Southern  1963   \n",
       "4              AFE               ZH  Africa Eastern and Southern  1964   \n",
       "5              AFE               ZH  Africa Eastern and Southern  1965   \n",
       "6              AFE               ZH  Africa Eastern and Southern  1966   \n",
       "7              AFE               ZH  Africa Eastern and Southern  1967   \n",
       "8              AFE               ZH  Africa Eastern and Southern  1968   \n",
       "9              AFE               ZH  Africa Eastern and Southern  1969   \n",
       "10             AFE               ZH  Africa Eastern and Southern  1970   \n",
       "11             AFE               ZH  Africa Eastern and Southern  1971   \n",
       "12             AFE               ZH  Africa Eastern and Southern  1972   \n",
       "13             AFE               ZH  Africa Eastern and Southern  1973   \n",
       "14             AFE               ZH  Africa Eastern and Southern  1974   \n",
       "15             AFE               ZH  Africa Eastern and Southern  1975   \n",
       "16             AFE               ZH  Africa Eastern and Southern  1976   \n",
       "17             AFE               ZH  Africa Eastern and Southern  1977   \n",
       "18             AFE               ZH  Africa Eastern and Southern  1978   \n",
       "19             AFE               ZH  Africa Eastern and Southern  1979   \n",
       "\n",
       "   indicator_code  indicator_value  \n",
       "0     IT.NET.BBND              NaN  \n",
       "1     IT.NET.BBND              NaN  \n",
       "2     IT.NET.BBND              NaN  \n",
       "3     IT.NET.BBND              NaN  \n",
       "4     IT.NET.BBND              NaN  \n",
       "5     IT.NET.BBND              NaN  \n",
       "6     IT.NET.BBND              NaN  \n",
       "7     IT.NET.BBND              NaN  \n",
       "8     IT.NET.BBND              NaN  \n",
       "9     IT.NET.BBND              NaN  \n",
       "10    IT.NET.BBND              NaN  \n",
       "11    IT.NET.BBND              NaN  \n",
       "12    IT.NET.BBND              NaN  \n",
       "13    IT.NET.BBND              NaN  \n",
       "14    IT.NET.BBND              NaN  \n",
       "15    IT.NET.BBND              NaN  \n",
       "16    IT.NET.BBND              NaN  \n",
       "17    IT.NET.BBND              NaN  \n",
       "18    IT.NET.BBND              NaN  \n",
       "19    IT.NET.BBND              NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "countryIndicators = indicatorsDF.join(countryDimFinal\n",
    "                                       , indicatorsDF.wb_country_code == countryDim.wb_country_code\n",
    "                                       , \"inner\") \\\n",
    "    .select(indicatorsDF.wb_country_code\n",
    "            , countryDim.country_iso_code\n",
    "            , countryDim.name\n",
    "            , indicatorsDF.year\n",
    "            , indicatorsDF.indicator_code\n",
    "            , indicatorsDF.indicator_value)\n",
    "\n",
    "showDF(countryIndicators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "39f5d474-4d35-48a4-a934-6d4c134e5d47",
   "metadata": {},
   "outputs": [],
   "source": [
    "countryPivot = countryIndicators.groupBy('country_iso_code', 'name', 'year') \\\n",
    "    .pivot('indicator_code').sum('indicator_value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "83f10888-de24-4070-bc77-a6a9cb852429",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>IT.CEL.SETS</th>\n",
       "      <th>IT.NET.BBND</th>\n",
       "      <th>SP.POP.0014.TO</th>\n",
       "      <th>SP.POP.1564.TO</th>\n",
       "      <th>SP.POP.65UP.TO</th>\n",
       "      <th>SP.POP.TOTL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1960</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.990027e+07</td>\n",
       "      <td>4.906322e+07</td>\n",
       "      <td>3234225.0</td>\n",
       "      <td>9.219772e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1961</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.133972e+07</td>\n",
       "      <td>5.003221e+07</td>\n",
       "      <td>3352609.0</td>\n",
       "      <td>9.472454e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.279288e+07</td>\n",
       "      <td>5.107209e+07</td>\n",
       "      <td>3469470.0</td>\n",
       "      <td>9.733444e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1963</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.424895e+07</td>\n",
       "      <td>5.220046e+07</td>\n",
       "      <td>3584776.0</td>\n",
       "      <td>1.000342e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.568530e+07</td>\n",
       "      <td>5.344922e+07</td>\n",
       "      <td>3698270.0</td>\n",
       "      <td>1.028328e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1965</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.708985e+07</td>\n",
       "      <td>5.483664e+07</td>\n",
       "      <td>3809935.0</td>\n",
       "      <td>1.057364e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1966</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.866839e+07</td>\n",
       "      <td>5.615092e+07</td>\n",
       "      <td>3939319.0</td>\n",
       "      <td>1.087586e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.018467e+07</td>\n",
       "      <td>5.764845e+07</td>\n",
       "      <td>4066217.0</td>\n",
       "      <td>1.118993e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.165773e+07</td>\n",
       "      <td>5.928679e+07</td>\n",
       "      <td>4191645.0</td>\n",
       "      <td>1.151362e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.310829e+07</td>\n",
       "      <td>6.101224e+07</td>\n",
       "      <td>4316656.0</td>\n",
       "      <td>1.184372e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.454215e+07</td>\n",
       "      <td>6.280120e+07</td>\n",
       "      <td>4442280.0</td>\n",
       "      <td>1.217856e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.613170e+07</td>\n",
       "      <td>6.444558e+07</td>\n",
       "      <td>4587447.0</td>\n",
       "      <td>1.251647e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1972</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.766822e+07</td>\n",
       "      <td>6.619852e+07</td>\n",
       "      <td>4732009.0</td>\n",
       "      <td>1.285987e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.920111e+07</td>\n",
       "      <td>6.808460e+07</td>\n",
       "      <td>4875588.0</td>\n",
       "      <td>1.321613e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1974</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.079685e+07</td>\n",
       "      <td>7.013819e+07</td>\n",
       "      <td>5017224.0</td>\n",
       "      <td>1.359523e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1975</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.250571e+07</td>\n",
       "      <td>7.237753e+07</td>\n",
       "      <td>5157335.0</td>\n",
       "      <td>1.400406e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.440912e+07</td>\n",
       "      <td>7.472836e+07</td>\n",
       "      <td>5315788.0</td>\n",
       "      <td>1.444533e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.641100e+07</td>\n",
       "      <td>7.727872e+07</td>\n",
       "      <td>5472123.0</td>\n",
       "      <td>1.491618e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1978</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.848723e+07</td>\n",
       "      <td>7.999613e+07</td>\n",
       "      <td>5627797.0</td>\n",
       "      <td>1.541112e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1979</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.058885e+07</td>\n",
       "      <td>8.284499e+07</td>\n",
       "      <td>5784698.0</td>\n",
       "      <td>1.592185e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.267635e+07</td>\n",
       "      <td>8.580016e+07</td>\n",
       "      <td>5944263.0</td>\n",
       "      <td>1.644208e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1981</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.487530e+07</td>\n",
       "      <td>8.869982e+07</td>\n",
       "      <td>6123853.0</td>\n",
       "      <td>1.696990e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1982</td>\n",
       "      <td>2330.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.704367e+07</td>\n",
       "      <td>9.171632e+07</td>\n",
       "      <td>6301811.0</td>\n",
       "      <td>1.750618e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1983</td>\n",
       "      <td>3564.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.918598e+07</td>\n",
       "      <td>9.483974e+07</td>\n",
       "      <td>6480241.0</td>\n",
       "      <td>1.805060e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1984</td>\n",
       "      <td>4936.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.131315e+07</td>\n",
       "      <td>9.806018e+07</td>\n",
       "      <td>6661954.0</td>\n",
       "      <td>1.860353e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1985</td>\n",
       "      <td>7976.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.342752e+07</td>\n",
       "      <td>1.013729e+08</td>\n",
       "      <td>6849912.0</td>\n",
       "      <td>1.916503e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1986</td>\n",
       "      <td>20756.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.573766e+07</td>\n",
       "      <td>1.045350e+08</td>\n",
       "      <td>7065460.0</td>\n",
       "      <td>1.973381e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1987</td>\n",
       "      <td>33793.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.797746e+07</td>\n",
       "      <td>1.078199e+08</td>\n",
       "      <td>7287560.0</td>\n",
       "      <td>2.030850e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1988</td>\n",
       "      <td>39447.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.016176e+07</td>\n",
       "      <td>1.112088e+08</td>\n",
       "      <td>7519114.0</td>\n",
       "      <td>2.088897e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1989</td>\n",
       "      <td>58750.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.228442e+07</td>\n",
       "      <td>1.147063e+08</td>\n",
       "      <td>7763206.0</td>\n",
       "      <td>2.147540e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1990</td>\n",
       "      <td>88618.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.527698e+07</td>\n",
       "      <td>1.193121e+08</td>\n",
       "      <td>8064277.0</td>\n",
       "      <td>2.226534e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1991</td>\n",
       "      <td>129904.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.724268e+07</td>\n",
       "      <td>1.231187e+08</td>\n",
       "      <td>8370338.0</td>\n",
       "      <td>2.287317e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1992</td>\n",
       "      <td>152281.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.846825e+07</td>\n",
       "      <td>1.258308e+08</td>\n",
       "      <td>8657337.0</td>\n",
       "      <td>2.329564e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1993</td>\n",
       "      <td>199350.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.002225e+08</td>\n",
       "      <td>1.300388e+08</td>\n",
       "      <td>8982024.0</td>\n",
       "      <td>2.392433e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1994</td>\n",
       "      <td>263401.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.017682e+08</td>\n",
       "      <td>1.343678e+08</td>\n",
       "      <td>9313402.0</td>\n",
       "      <td>2.454494e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1995</td>\n",
       "      <td>501551.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.035753e+08</td>\n",
       "      <td>1.398595e+08</td>\n",
       "      <td>9672492.0</td>\n",
       "      <td>2.531073e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1996</td>\n",
       "      <td>921229.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.048058e+08</td>\n",
       "      <td>1.441792e+08</td>\n",
       "      <td>10015939.0</td>\n",
       "      <td>2.590009e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1997</td>\n",
       "      <td>1616551.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.058867e+08</td>\n",
       "      <td>1.485823e+08</td>\n",
       "      <td>10353214.0</td>\n",
       "      <td>2.648222e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1998</td>\n",
       "      <td>2523542.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.068452e+08</td>\n",
       "      <td>1.530452e+08</td>\n",
       "      <td>10685323.0</td>\n",
       "      <td>2.705758e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>1999</td>\n",
       "      <td>4105508.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.077524e+08</td>\n",
       "      <td>1.576263e+08</td>\n",
       "      <td>11015111.0</td>\n",
       "      <td>2.763938e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2000</td>\n",
       "      <td>9035241.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.086316e+08</td>\n",
       "      <td>1.623677e+08</td>\n",
       "      <td>11344828.0</td>\n",
       "      <td>2.823441e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2001</td>\n",
       "      <td>16679594.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.094353e+08</td>\n",
       "      <td>1.673248e+08</td>\n",
       "      <td>11672023.0</td>\n",
       "      <td>2.884322e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2002</td>\n",
       "      <td>25275663.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.102056e+08</td>\n",
       "      <td>1.724625e+08</td>\n",
       "      <td>11997114.0</td>\n",
       "      <td>2.946652e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2003</td>\n",
       "      <td>35121269.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.109946e+08</td>\n",
       "      <td>1.777990e+08</td>\n",
       "      <td>12320307.0</td>\n",
       "      <td>3.011139e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2004</td>\n",
       "      <td>52132873.0</td>\n",
       "      <td>365937.0</td>\n",
       "      <td>1.118714e+08</td>\n",
       "      <td>1.833530e+08</td>\n",
       "      <td>12638464.0</td>\n",
       "      <td>3.078628e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2005</td>\n",
       "      <td>84854418.0</td>\n",
       "      <td>861800.0</td>\n",
       "      <td>1.128867e+08</td>\n",
       "      <td>1.891263e+08</td>\n",
       "      <td>12952804.0</td>\n",
       "      <td>3.149658e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2006</td>\n",
       "      <td>125958922.0</td>\n",
       "      <td>1545499.0</td>\n",
       "      <td>1.139795e+08</td>\n",
       "      <td>1.951837e+08</td>\n",
       "      <td>13289587.0</td>\n",
       "      <td>3.224528e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2007</td>\n",
       "      <td>174488139.0</td>\n",
       "      <td>2811961.0</td>\n",
       "      <td>1.152315e+08</td>\n",
       "      <td>2.014426e+08</td>\n",
       "      <td>13616630.0</td>\n",
       "      <td>3.302908e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2008</td>\n",
       "      <td>214062549.0</td>\n",
       "      <td>4235468.0</td>\n",
       "      <td>1.166618e+08</td>\n",
       "      <td>2.077879e+08</td>\n",
       "      <td>13946302.0</td>\n",
       "      <td>3.383959e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2009</td>\n",
       "      <td>265298629.0</td>\n",
       "      <td>5845325.0</td>\n",
       "      <td>1.182674e+08</td>\n",
       "      <td>2.140688e+08</td>\n",
       "      <td>14292986.0</td>\n",
       "      <td>3.466292e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2010</td>\n",
       "      <td>312247800.0</td>\n",
       "      <td>7123797.0</td>\n",
       "      <td>1.200390e+08</td>\n",
       "      <td>2.201833e+08</td>\n",
       "      <td>14667787.0</td>\n",
       "      <td>3.548901e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2011</td>\n",
       "      <td>351957968.0</td>\n",
       "      <td>8570787.0</td>\n",
       "      <td>1.219339e+08</td>\n",
       "      <td>2.261422e+08</td>\n",
       "      <td>15080808.0</td>\n",
       "      <td>3.631568e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2012</td>\n",
       "      <td>381641858.0</td>\n",
       "      <td>10323586.0</td>\n",
       "      <td>1.240056e+08</td>\n",
       "      <td>2.319236e+08</td>\n",
       "      <td>15508463.0</td>\n",
       "      <td>3.714376e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2013</td>\n",
       "      <td>407704505.0</td>\n",
       "      <td>8880291.0</td>\n",
       "      <td>1.262106e+08</td>\n",
       "      <td>2.375227e+08</td>\n",
       "      <td>15963198.0</td>\n",
       "      <td>3.796965e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2014</td>\n",
       "      <td>415029432.0</td>\n",
       "      <td>10801834.0</td>\n",
       "      <td>1.284472e+08</td>\n",
       "      <td>2.429867e+08</td>\n",
       "      <td>16465991.0</td>\n",
       "      <td>3.878998e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2015</td>\n",
       "      <td>419013963.0</td>\n",
       "      <td>18081304.0</td>\n",
       "      <td>1.306295e+08</td>\n",
       "      <td>2.483654e+08</td>\n",
       "      <td>17033363.0</td>\n",
       "      <td>3.960283e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2016</td>\n",
       "      <td>417212258.0</td>\n",
       "      <td>20019879.0</td>\n",
       "      <td>1.331965e+08</td>\n",
       "      <td>2.531713e+08</td>\n",
       "      <td>17675107.0</td>\n",
       "      <td>4.040429e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2017</td>\n",
       "      <td>416484374.0</td>\n",
       "      <td>28218281.0</td>\n",
       "      <td>1.354823e+08</td>\n",
       "      <td>2.580814e+08</td>\n",
       "      <td>18379135.0</td>\n",
       "      <td>4.119428e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2018</td>\n",
       "      <td>411117427.0</td>\n",
       "      <td>30067212.0</td>\n",
       "      <td>1.376280e+08</td>\n",
       "      <td>2.630875e+08</td>\n",
       "      <td>19136464.0</td>\n",
       "      <td>4.198520e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2019</td>\n",
       "      <td>421459796.0</td>\n",
       "      <td>33114238.0</td>\n",
       "      <td>1.397829e+08</td>\n",
       "      <td>2.681579e+08</td>\n",
       "      <td>19929502.0</td>\n",
       "      <td>4.278703e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2020</td>\n",
       "      <td>423344019.0</td>\n",
       "      <td>38898085.0</td>\n",
       "      <td>1.420035e+08</td>\n",
       "      <td>2.733236e+08</td>\n",
       "      <td>20753658.0</td>\n",
       "      <td>4.360807e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>1A</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>2021</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.440012e+08</td>\n",
       "      <td>2.788425e+08</td>\n",
       "      <td>21674036.0</td>\n",
       "      <td>4.445178e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1960</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.126566e+09</td>\n",
       "      <td>1.753555e+09</td>\n",
       "      <td>150633690.0</td>\n",
       "      <td>3.032156e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1961</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.146759e+09</td>\n",
       "      <td>1.769384e+09</td>\n",
       "      <td>154023180.0</td>\n",
       "      <td>3.071596e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.173476e+09</td>\n",
       "      <td>1.792141e+09</td>\n",
       "      <td>157478382.0</td>\n",
       "      <td>3.124561e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1963</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.204620e+09</td>\n",
       "      <td>1.822466e+09</td>\n",
       "      <td>161066505.0</td>\n",
       "      <td>3.189656e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.233296e+09</td>\n",
       "      <td>1.855792e+09</td>\n",
       "      <td>164516743.0</td>\n",
       "      <td>3.255146e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1965</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.259110e+09</td>\n",
       "      <td>1.893424e+09</td>\n",
       "      <td>167934244.0</td>\n",
       "      <td>3.322047e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1966</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.288632e+09</td>\n",
       "      <td>1.928452e+09</td>\n",
       "      <td>173397291.0</td>\n",
       "      <td>3.392098e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.313422e+09</td>\n",
       "      <td>1.967722e+09</td>\n",
       "      <td>178820397.0</td>\n",
       "      <td>3.461620e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.335822e+09</td>\n",
       "      <td>2.011010e+09</td>\n",
       "      <td>184257318.0</td>\n",
       "      <td>3.532783e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.358383e+09</td>\n",
       "      <td>2.056731e+09</td>\n",
       "      <td>189708271.0</td>\n",
       "      <td>3.606554e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.381778e+09</td>\n",
       "      <td>2.103319e+09</td>\n",
       "      <td>195108426.0</td>\n",
       "      <td>3.681976e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.408244e+09</td>\n",
       "      <td>2.148672e+09</td>\n",
       "      <td>201794523.0</td>\n",
       "      <td>3.760517e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1972</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.433034e+09</td>\n",
       "      <td>2.193875e+09</td>\n",
       "      <td>208145557.0</td>\n",
       "      <td>3.836901e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.456625e+09</td>\n",
       "      <td>2.240153e+09</td>\n",
       "      <td>214321972.0</td>\n",
       "      <td>3.912984e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1974</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.477990e+09</td>\n",
       "      <td>2.288129e+09</td>\n",
       "      <td>220445210.0</td>\n",
       "      <td>3.988487e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1975</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.496155e+09</td>\n",
       "      <td>2.337851e+09</td>\n",
       "      <td>226540705.0</td>\n",
       "      <td>4.062507e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.514176e+09</td>\n",
       "      <td>2.385362e+09</td>\n",
       "      <td>233893891.0</td>\n",
       "      <td>4.135432e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.529015e+09</td>\n",
       "      <td>2.435474e+09</td>\n",
       "      <td>241260067.0</td>\n",
       "      <td>4.207786e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1978</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.541827e+09</td>\n",
       "      <td>2.488945e+09</td>\n",
       "      <td>248491687.0</td>\n",
       "      <td>4.281339e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1979</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.553828e+09</td>\n",
       "      <td>2.545630e+09</td>\n",
       "      <td>255204029.0</td>\n",
       "      <td>4.356778e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1980</td>\n",
       "      <td>23482.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.565443e+09</td>\n",
       "      <td>2.604263e+09</td>\n",
       "      <td>261101284.0</td>\n",
       "      <td>4.432964e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1981</td>\n",
       "      <td>63585.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.577945e+09</td>\n",
       "      <td>2.663755e+09</td>\n",
       "      <td>267266455.0</td>\n",
       "      <td>4.511164e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1982</td>\n",
       "      <td>101509.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.592069e+09</td>\n",
       "      <td>2.725600e+09</td>\n",
       "      <td>272479750.0</td>\n",
       "      <td>4.592387e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1983</td>\n",
       "      <td>148855.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.606576e+09</td>\n",
       "      <td>2.788328e+09</td>\n",
       "      <td>277144402.0</td>\n",
       "      <td>4.674330e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1984</td>\n",
       "      <td>319252.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.620614e+09</td>\n",
       "      <td>2.851151e+09</td>\n",
       "      <td>281902607.0</td>\n",
       "      <td>4.755997e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1985</td>\n",
       "      <td>750629.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.634833e+09</td>\n",
       "      <td>2.914715e+09</td>\n",
       "      <td>287249869.0</td>\n",
       "      <td>4.839177e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1986</td>\n",
       "      <td>1450136.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.655397e+09</td>\n",
       "      <td>2.972772e+09</td>\n",
       "      <td>294144793.0</td>\n",
       "      <td>4.924748e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1987</td>\n",
       "      <td>2545270.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.675814e+09</td>\n",
       "      <td>3.032844e+09</td>\n",
       "      <td>301407523.0</td>\n",
       "      <td>5.012556e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1988</td>\n",
       "      <td>4327956.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.695923e+09</td>\n",
       "      <td>3.093810e+09</td>\n",
       "      <td>309002855.0</td>\n",
       "      <td>5.101288e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1989</td>\n",
       "      <td>7353071.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.715226e+09</td>\n",
       "      <td>3.155232e+09</td>\n",
       "      <td>316910626.0</td>\n",
       "      <td>5.189977e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1990</td>\n",
       "      <td>11210115.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.734022e+09</td>\n",
       "      <td>3.218211e+09</td>\n",
       "      <td>325168639.0</td>\n",
       "      <td>5.280063e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1991</td>\n",
       "      <td>16276503.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.756500e+09</td>\n",
       "      <td>3.274403e+09</td>\n",
       "      <td>334528596.0</td>\n",
       "      <td>5.368140e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1992</td>\n",
       "      <td>23245154.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.773095e+09</td>\n",
       "      <td>3.332572e+09</td>\n",
       "      <td>344160101.0</td>\n",
       "      <td>5.452577e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1993</td>\n",
       "      <td>34163482.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.786860e+09</td>\n",
       "      <td>3.394342e+09</td>\n",
       "      <td>353893707.0</td>\n",
       "      <td>5.537885e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1994</td>\n",
       "      <td>55511640.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.799326e+09</td>\n",
       "      <td>3.456641e+09</td>\n",
       "      <td>363289219.0</td>\n",
       "      <td>5.622085e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1995</td>\n",
       "      <td>90743607.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.811875e+09</td>\n",
       "      <td>3.519786e+09</td>\n",
       "      <td>372224770.0</td>\n",
       "      <td>5.706754e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1996</td>\n",
       "      <td>145042295.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.818373e+09</td>\n",
       "      <td>3.585670e+09</td>\n",
       "      <td>382704184.0</td>\n",
       "      <td>5.789655e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1W</td>\n",
       "      <td>World</td>\n",
       "      <td>1997</td>\n",
       "      <td>214953349.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.827836e+09</td>\n",
       "      <td>3.649230e+09</td>\n",
       "      <td>392273082.0</td>\n",
       "      <td>5.872284e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_iso_code        name  year  IT.CEL.SETS  IT.NET.BBND  \\\n",
       "0                1A  Arab World  1960          0.0          NaN   \n",
       "1                1A  Arab World  1961          NaN          NaN   \n",
       "2                1A  Arab World  1962          NaN          NaN   \n",
       "3                1A  Arab World  1963          NaN          NaN   \n",
       "4                1A  Arab World  1964          NaN          NaN   \n",
       "5                1A  Arab World  1965          0.0          NaN   \n",
       "6                1A  Arab World  1966          NaN          NaN   \n",
       "7                1A  Arab World  1967          NaN          NaN   \n",
       "8                1A  Arab World  1968          NaN          NaN   \n",
       "9                1A  Arab World  1969          NaN          NaN   \n",
       "10               1A  Arab World  1970          0.0          NaN   \n",
       "11               1A  Arab World  1971          NaN          NaN   \n",
       "12               1A  Arab World  1972          NaN          NaN   \n",
       "13               1A  Arab World  1973          NaN          NaN   \n",
       "14               1A  Arab World  1974          NaN          NaN   \n",
       "15               1A  Arab World  1975          0.0          NaN   \n",
       "16               1A  Arab World  1976          0.0          NaN   \n",
       "17               1A  Arab World  1977          0.0          NaN   \n",
       "18               1A  Arab World  1978          0.0          NaN   \n",
       "19               1A  Arab World  1979          0.0          NaN   \n",
       "20               1A  Arab World  1980          0.0          NaN   \n",
       "21               1A  Arab World  1981          0.0          NaN   \n",
       "22               1A  Arab World  1982       2330.0          NaN   \n",
       "23               1A  Arab World  1983       3564.0          NaN   \n",
       "24               1A  Arab World  1984       4936.0          NaN   \n",
       "25               1A  Arab World  1985       7976.0          NaN   \n",
       "26               1A  Arab World  1986      20756.0          NaN   \n",
       "27               1A  Arab World  1987      33793.0          NaN   \n",
       "28               1A  Arab World  1988      39447.0          NaN   \n",
       "29               1A  Arab World  1989      58750.0          NaN   \n",
       "30               1A  Arab World  1990      88618.0          NaN   \n",
       "31               1A  Arab World  1991     129904.0          NaN   \n",
       "32               1A  Arab World  1992     152281.0          NaN   \n",
       "33               1A  Arab World  1993     199350.0          NaN   \n",
       "34               1A  Arab World  1994     263401.0          NaN   \n",
       "35               1A  Arab World  1995     501551.0          NaN   \n",
       "36               1A  Arab World  1996     921229.0          NaN   \n",
       "37               1A  Arab World  1997    1616551.0          NaN   \n",
       "38               1A  Arab World  1998    2523542.0          NaN   \n",
       "39               1A  Arab World  1999    4105508.0          NaN   \n",
       "40               1A  Arab World  2000    9035241.0          NaN   \n",
       "41               1A  Arab World  2001   16679594.0          NaN   \n",
       "42               1A  Arab World  2002   25275663.0          NaN   \n",
       "43               1A  Arab World  2003   35121269.0          NaN   \n",
       "44               1A  Arab World  2004   52132873.0     365937.0   \n",
       "45               1A  Arab World  2005   84854418.0     861800.0   \n",
       "46               1A  Arab World  2006  125958922.0    1545499.0   \n",
       "47               1A  Arab World  2007  174488139.0    2811961.0   \n",
       "48               1A  Arab World  2008  214062549.0    4235468.0   \n",
       "49               1A  Arab World  2009  265298629.0    5845325.0   \n",
       "50               1A  Arab World  2010  312247800.0    7123797.0   \n",
       "51               1A  Arab World  2011  351957968.0    8570787.0   \n",
       "52               1A  Arab World  2012  381641858.0   10323586.0   \n",
       "53               1A  Arab World  2013  407704505.0    8880291.0   \n",
       "54               1A  Arab World  2014  415029432.0   10801834.0   \n",
       "55               1A  Arab World  2015  419013963.0   18081304.0   \n",
       "56               1A  Arab World  2016  417212258.0   20019879.0   \n",
       "57               1A  Arab World  2017  416484374.0   28218281.0   \n",
       "58               1A  Arab World  2018  411117427.0   30067212.0   \n",
       "59               1A  Arab World  2019  421459796.0   33114238.0   \n",
       "60               1A  Arab World  2020  423344019.0   38898085.0   \n",
       "61               1A  Arab World  2021          NaN          NaN   \n",
       "62               1W       World  1960          0.0          NaN   \n",
       "63               1W       World  1961          NaN          NaN   \n",
       "64               1W       World  1962          NaN          NaN   \n",
       "65               1W       World  1963          NaN          NaN   \n",
       "66               1W       World  1964          NaN          NaN   \n",
       "67               1W       World  1965          0.0          NaN   \n",
       "68               1W       World  1966          NaN          NaN   \n",
       "69               1W       World  1967          NaN          NaN   \n",
       "70               1W       World  1968          NaN          NaN   \n",
       "71               1W       World  1969          NaN          NaN   \n",
       "72               1W       World  1970          0.0          NaN   \n",
       "73               1W       World  1971          NaN          NaN   \n",
       "74               1W       World  1972          NaN          NaN   \n",
       "75               1W       World  1973          NaN          NaN   \n",
       "76               1W       World  1974          NaN          NaN   \n",
       "77               1W       World  1975          0.0          NaN   \n",
       "78               1W       World  1976          0.0          NaN   \n",
       "79               1W       World  1977          0.0          NaN   \n",
       "80               1W       World  1978          0.0          NaN   \n",
       "81               1W       World  1979          0.0          NaN   \n",
       "82               1W       World  1980      23482.0          NaN   \n",
       "83               1W       World  1981      63585.0          NaN   \n",
       "84               1W       World  1982     101509.0          NaN   \n",
       "85               1W       World  1983     148855.0          NaN   \n",
       "86               1W       World  1984     319252.0          NaN   \n",
       "87               1W       World  1985     750629.0          NaN   \n",
       "88               1W       World  1986    1450136.0          NaN   \n",
       "89               1W       World  1987    2545270.0          NaN   \n",
       "90               1W       World  1988    4327956.0          NaN   \n",
       "91               1W       World  1989    7353071.0          NaN   \n",
       "92               1W       World  1990   11210115.0          NaN   \n",
       "93               1W       World  1991   16276503.0          NaN   \n",
       "94               1W       World  1992   23245154.0          NaN   \n",
       "95               1W       World  1993   34163482.0          NaN   \n",
       "96               1W       World  1994   55511640.0          NaN   \n",
       "97               1W       World  1995   90743607.0          NaN   \n",
       "98               1W       World  1996  145042295.0          NaN   \n",
       "99               1W       World  1997  214953349.0          NaN   \n",
       "\n",
       "    SP.POP.0014.TO  SP.POP.1564.TO  SP.POP.65UP.TO   SP.POP.TOTL  \n",
       "0     3.990027e+07    4.906322e+07       3234225.0  9.219772e+07  \n",
       "1     4.133972e+07    5.003221e+07       3352609.0  9.472454e+07  \n",
       "2     4.279288e+07    5.107209e+07       3469470.0  9.733444e+07  \n",
       "3     4.424895e+07    5.220046e+07       3584776.0  1.000342e+08  \n",
       "4     4.568530e+07    5.344922e+07       3698270.0  1.028328e+08  \n",
       "5     4.708985e+07    5.483664e+07       3809935.0  1.057364e+08  \n",
       "6     4.866839e+07    5.615092e+07       3939319.0  1.087586e+08  \n",
       "7     5.018467e+07    5.764845e+07       4066217.0  1.118993e+08  \n",
       "8     5.165773e+07    5.928679e+07       4191645.0  1.151362e+08  \n",
       "9     5.310829e+07    6.101224e+07       4316656.0  1.184372e+08  \n",
       "10    5.454215e+07    6.280120e+07       4442280.0  1.217856e+08  \n",
       "11    5.613170e+07    6.444558e+07       4587447.0  1.251647e+08  \n",
       "12    5.766822e+07    6.619852e+07       4732009.0  1.285987e+08  \n",
       "13    5.920111e+07    6.808460e+07       4875588.0  1.321613e+08  \n",
       "14    6.079685e+07    7.013819e+07       5017224.0  1.359523e+08  \n",
       "15    6.250571e+07    7.237753e+07       5157335.0  1.400406e+08  \n",
       "16    6.440912e+07    7.472836e+07       5315788.0  1.444533e+08  \n",
       "17    6.641100e+07    7.727872e+07       5472123.0  1.491618e+08  \n",
       "18    6.848723e+07    7.999613e+07       5627797.0  1.541112e+08  \n",
       "19    7.058885e+07    8.284499e+07       5784698.0  1.592185e+08  \n",
       "20    7.267635e+07    8.580016e+07       5944263.0  1.644208e+08  \n",
       "21    7.487530e+07    8.869982e+07       6123853.0  1.696990e+08  \n",
       "22    7.704367e+07    9.171632e+07       6301811.0  1.750618e+08  \n",
       "23    7.918598e+07    9.483974e+07       6480241.0  1.805060e+08  \n",
       "24    8.131315e+07    9.806018e+07       6661954.0  1.860353e+08  \n",
       "25    8.342752e+07    1.013729e+08       6849912.0  1.916503e+08  \n",
       "26    8.573766e+07    1.045350e+08       7065460.0  1.973381e+08  \n",
       "27    8.797746e+07    1.078199e+08       7287560.0  2.030850e+08  \n",
       "28    9.016176e+07    1.112088e+08       7519114.0  2.088897e+08  \n",
       "29    9.228442e+07    1.147063e+08       7763206.0  2.147540e+08  \n",
       "30    9.527698e+07    1.193121e+08       8064277.0  2.226534e+08  \n",
       "31    9.724268e+07    1.231187e+08       8370338.0  2.287317e+08  \n",
       "32    9.846825e+07    1.258308e+08       8657337.0  2.329564e+08  \n",
       "33    1.002225e+08    1.300388e+08       8982024.0  2.392433e+08  \n",
       "34    1.017682e+08    1.343678e+08       9313402.0  2.454494e+08  \n",
       "35    1.035753e+08    1.398595e+08       9672492.0  2.531073e+08  \n",
       "36    1.048058e+08    1.441792e+08      10015939.0  2.590009e+08  \n",
       "37    1.058867e+08    1.485823e+08      10353214.0  2.648222e+08  \n",
       "38    1.068452e+08    1.530452e+08      10685323.0  2.705758e+08  \n",
       "39    1.077524e+08    1.576263e+08      11015111.0  2.763938e+08  \n",
       "40    1.086316e+08    1.623677e+08      11344828.0  2.823441e+08  \n",
       "41    1.094353e+08    1.673248e+08      11672023.0  2.884322e+08  \n",
       "42    1.102056e+08    1.724625e+08      11997114.0  2.946652e+08  \n",
       "43    1.109946e+08    1.777990e+08      12320307.0  3.011139e+08  \n",
       "44    1.118714e+08    1.833530e+08      12638464.0  3.078628e+08  \n",
       "45    1.128867e+08    1.891263e+08      12952804.0  3.149658e+08  \n",
       "46    1.139795e+08    1.951837e+08      13289587.0  3.224528e+08  \n",
       "47    1.152315e+08    2.014426e+08      13616630.0  3.302908e+08  \n",
       "48    1.166618e+08    2.077879e+08      13946302.0  3.383959e+08  \n",
       "49    1.182674e+08    2.140688e+08      14292986.0  3.466292e+08  \n",
       "50    1.200390e+08    2.201833e+08      14667787.0  3.548901e+08  \n",
       "51    1.219339e+08    2.261422e+08      15080808.0  3.631568e+08  \n",
       "52    1.240056e+08    2.319236e+08      15508463.0  3.714376e+08  \n",
       "53    1.262106e+08    2.375227e+08      15963198.0  3.796965e+08  \n",
       "54    1.284472e+08    2.429867e+08      16465991.0  3.878998e+08  \n",
       "55    1.306295e+08    2.483654e+08      17033363.0  3.960283e+08  \n",
       "56    1.331965e+08    2.531713e+08      17675107.0  4.040429e+08  \n",
       "57    1.354823e+08    2.580814e+08      18379135.0  4.119428e+08  \n",
       "58    1.376280e+08    2.630875e+08      19136464.0  4.198520e+08  \n",
       "59    1.397829e+08    2.681579e+08      19929502.0  4.278703e+08  \n",
       "60    1.420035e+08    2.733236e+08      20753658.0  4.360807e+08  \n",
       "61    1.440012e+08    2.788425e+08      21674036.0  4.445178e+08  \n",
       "62    1.126566e+09    1.753555e+09     150633690.0  3.032156e+09  \n",
       "63    1.146759e+09    1.769384e+09     154023180.0  3.071596e+09  \n",
       "64    1.173476e+09    1.792141e+09     157478382.0  3.124561e+09  \n",
       "65    1.204620e+09    1.822466e+09     161066505.0  3.189656e+09  \n",
       "66    1.233296e+09    1.855792e+09     164516743.0  3.255146e+09  \n",
       "67    1.259110e+09    1.893424e+09     167934244.0  3.322047e+09  \n",
       "68    1.288632e+09    1.928452e+09     173397291.0  3.392098e+09  \n",
       "69    1.313422e+09    1.967722e+09     178820397.0  3.461620e+09  \n",
       "70    1.335822e+09    2.011010e+09     184257318.0  3.532783e+09  \n",
       "71    1.358383e+09    2.056731e+09     189708271.0  3.606554e+09  \n",
       "72    1.381778e+09    2.103319e+09     195108426.0  3.681976e+09  \n",
       "73    1.408244e+09    2.148672e+09     201794523.0  3.760517e+09  \n",
       "74    1.433034e+09    2.193875e+09     208145557.0  3.836901e+09  \n",
       "75    1.456625e+09    2.240153e+09     214321972.0  3.912984e+09  \n",
       "76    1.477990e+09    2.288129e+09     220445210.0  3.988487e+09  \n",
       "77    1.496155e+09    2.337851e+09     226540705.0  4.062507e+09  \n",
       "78    1.514176e+09    2.385362e+09     233893891.0  4.135432e+09  \n",
       "79    1.529015e+09    2.435474e+09     241260067.0  4.207786e+09  \n",
       "80    1.541827e+09    2.488945e+09     248491687.0  4.281339e+09  \n",
       "81    1.553828e+09    2.545630e+09     255204029.0  4.356778e+09  \n",
       "82    1.565443e+09    2.604263e+09     261101284.0  4.432964e+09  \n",
       "83    1.577945e+09    2.663755e+09     267266455.0  4.511164e+09  \n",
       "84    1.592069e+09    2.725600e+09     272479750.0  4.592387e+09  \n",
       "85    1.606576e+09    2.788328e+09     277144402.0  4.674330e+09  \n",
       "86    1.620614e+09    2.851151e+09     281902607.0  4.755997e+09  \n",
       "87    1.634833e+09    2.914715e+09     287249869.0  4.839177e+09  \n",
       "88    1.655397e+09    2.972772e+09     294144793.0  4.924748e+09  \n",
       "89    1.675814e+09    3.032844e+09     301407523.0  5.012556e+09  \n",
       "90    1.695923e+09    3.093810e+09     309002855.0  5.101288e+09  \n",
       "91    1.715226e+09    3.155232e+09     316910626.0  5.189977e+09  \n",
       "92    1.734022e+09    3.218211e+09     325168639.0  5.280063e+09  \n",
       "93    1.756500e+09    3.274403e+09     334528596.0  5.368140e+09  \n",
       "94    1.773095e+09    3.332572e+09     344160101.0  5.452577e+09  \n",
       "95    1.786860e+09    3.394342e+09     353893707.0  5.537885e+09  \n",
       "96    1.799326e+09    3.456641e+09     363289219.0  5.622085e+09  \n",
       "97    1.811875e+09    3.519786e+09     372224770.0  5.706754e+09  \n",
       "98    1.818373e+09    3.585670e+09     382704184.0  5.789655e+09  \n",
       "99    1.827836e+09    3.649230e+09     392273082.0  5.872284e+09  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showDF(countryPivot.orderBy('country_iso_code', 'name', 'year'), limitRows=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "b16126b0-bb72-4184-8bf0-14164ecaa04d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#Write the country-yearly totals to a CSV File\n",
    "countryPivot.filter('country_iso_code is not null') \\\n",
    "    .orderBy('country_iso_code','name', 'year') \\\n",
    "    .withColumnRenamed('IT.CEL.SETS', 'cellular_subscriptions') \\\n",
    "    .withColumnRenamed('IT.NET.BBND', 'broadband_subscriptions') \\\n",
    "    .withColumnRenamed('SP.POP.0014.TO', 'population_age_0_to_14') \\\n",
    "    .withColumnRenamed('SP.POP.1564.TO', 'population_age_15_64') \\\n",
    "    .withColumnRenamed('SP.POP.65UP.TO', 'population_age_65_and_above') \\\n",
    "    .withColumnRenamed('SP.POP.TOTL', 'population') \\\n",
    "    .select('country_iso_code'\n",
    "            , 'name'\n",
    "            , 'year'\n",
    "            , F.col('population').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_0_to_14').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_15_64').cast(DecimalType(38, 2))\n",
    "            , F.col('population_age_65_and_above').cast(DecimalType(38, 2))\n",
    "            , F.col('broadband_subscriptions').cast(DecimalType(38, 2))\n",
    "            , F.col('cellular_subscriptions').cast(DecimalType(38, 2))) \\\n",
    "    .coalesce(1) \\\n",
    "    .write.csv('output/CountryStats', mode='overwrite', header='true')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60a24a82",
   "metadata": {},
   "source": [
    "Business Environment Analysis:\n",
    "\n",
    "Kat wants to identify the countries that are conducive to starting a business. She is interested in the most recent metrics for the following indicators:\n",
    "\n",
    "Gross National Income (GNI)\n",
    "\n",
    "Cost of business start-up procedures\n",
    "\n",
    "Number of days required to start a business\n",
    "\n",
    "Number of start-up procedures to register a business\n",
    "\n",
    "GDP\n",
    "\n",
    "GDP per capita\n",
    "\n",
    "Business Regulatory Environment\n",
    "\n",
    "Ease of doing business index (available only for 2017)\n",
    "\n",
    "The data should be written to a CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "4d6efe3a-f11c-42c5-8af8-e29bca6a1686",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_name</th>\n",
       "      <th>periodicity</th>\n",
       "      <th>Aggregation_method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>Cost of business start-up procedures (% of GNI...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>Time required to start a business (days)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>Start-up procedures to register a business (nu...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>CPIA business regulatory environment rating (1...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>GDP (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>GDP per capita (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>GNI, Atlas method (current US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      indicator_code                                     indicator_name  \\\n",
       "0  IC.REG.COST.PC.ZS  Cost of business start-up procedures (% of GNI...   \n",
       "1        IC.REG.DURS           Time required to start a business (days)   \n",
       "2        IC.REG.PROC  Start-up procedures to register a business (nu...   \n",
       "3     IQ.CPA.BREG.XQ  CPIA business regulatory environment rating (1...   \n",
       "4     NY.GDP.MKTP.KD                            GDP (constant 2015 US$)   \n",
       "5     NY.GDP.PCAP.KD                 GDP per capita (constant 2015 US$)   \n",
       "6     NY.GNP.ATLS.CD                    GNI, Atlas method (current US$)   \n",
       "\n",
       "  periodicity  Aggregation_method  \n",
       "0      Annual  Unweighted average  \n",
       "1      Annual  Unweighted average  \n",
       "2      Annual  Unweighted average  \n",
       "3      Annual  Unweighted average  \n",
       "4      Annual    Gap-filled total  \n",
       "5      Annual    Weighted average  \n",
       "6      Annual    Gap-filled total  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "businessIndicators = seriesDim.filter(\n",
    "    (F.lower(F.col(\"indicator_name\")).like(\"%start%up%procedures%\") |\n",
    "     F.lower(F.col(\"indicator_name\")).like(\"%start%business%\") |\n",
    "     F.lower(F.col(\"indicator_name\")).like(\"%business%regulatory%environment%\") |\n",
    "     F.lower(F.col(\"indicator_name\")).like(\"%gdp%constant%us%\") |\n",
    "     F.lower(F.col(\"indicator_name\")).like(\"%gdp%per%capita%constant%us%\") |\n",
    "     F.lower(F.col(\"indicator_name\")).like(\"%gni, atlas%us%\")) &\n",
    "    ~F.lower(F.col(\"indicator_name\")).contains(\", female\") &\n",
    "    ~F.lower(F.col(\"indicator_name\")).contains(\", male\")\n",
    ")\n",
    "showDF(businessIndicators)\n",
    "businessIndicators.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "4cbc88ed-a4ca-44c4-97cb-42b01b3d04df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>1960</th>\n",
       "      <th>1961</th>\n",
       "      <th>1962</th>\n",
       "      <th>1963</th>\n",
       "      <th>1964</th>\n",
       "      <th>1965</th>\n",
       "      <th>1966</th>\n",
       "      <th>1967</th>\n",
       "      <th>...</th>\n",
       "      <th>2015</th>\n",
       "      <th>2016</th>\n",
       "      <th>2017</th>\n",
       "      <th>2018</th>\n",
       "      <th>2019</th>\n",
       "      <th>2020</th>\n",
       "      <th>2021</th>\n",
       "      <th>indicator_name</th>\n",
       "      <th>periodicity</th>\n",
       "      <th>Aggregation_method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>5.509615e+01</td>\n",
       "      <td>5.563077e+01</td>\n",
       "      <td>5.107692e+01</td>\n",
       "      <td>4.173846e+01</td>\n",
       "      <td>3.423077e+01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cost of business start-up procedures (% of GNI...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>3.027778e+00</td>\n",
       "      <td>3.027778e+00</td>\n",
       "      <td>2.973684e+00</td>\n",
       "      <td>3.027778e+00</td>\n",
       "      <td>2.921053e+00</td>\n",
       "      <td>2.921053e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CPIA business regulatory environment rating (1...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>1.538294e+11</td>\n",
       "      <td>1.541980e+11</td>\n",
       "      <td>1.665041e+11</td>\n",
       "      <td>1.751030e+11</td>\n",
       "      <td>1.831226e+11</td>\n",
       "      <td>1.928789e+11</td>\n",
       "      <td>2.004245e+11</td>\n",
       "      <td>2.109604e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>9.242525e+11</td>\n",
       "      <td>9.447434e+11</td>\n",
       "      <td>9.689017e+11</td>\n",
       "      <td>9.929813e+11</td>\n",
       "      <td>1.013099e+12</td>\n",
       "      <td>9.838518e+11</td>\n",
       "      <td>1.026194e+12</td>\n",
       "      <td>GDP (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>1.175735e+03</td>\n",
       "      <td>1.149361e+03</td>\n",
       "      <td>1.209930e+03</td>\n",
       "      <td>1.240088e+03</td>\n",
       "      <td>1.263610e+03</td>\n",
       "      <td>1.296491e+03</td>\n",
       "      <td>1.312085e+03</td>\n",
       "      <td>1.344755e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>1.556316e+03</td>\n",
       "      <td>1.548813e+03</td>\n",
       "      <td>1.546796e+03</td>\n",
       "      <td>1.544078e+03</td>\n",
       "      <td>1.534890e+03</td>\n",
       "      <td>1.452730e+03</td>\n",
       "      <td>1.477249e+03</td>\n",
       "      <td>GDP per capita (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9.476731e+11</td>\n",
       "      <td>9.003166e+11</td>\n",
       "      <td>9.142314e+11</td>\n",
       "      <td>9.421241e+11</td>\n",
       "      <td>9.945128e+11</td>\n",
       "      <td>9.464769e+11</td>\n",
       "      <td>1.025115e+12</td>\n",
       "      <td>GNI, Atlas method (current US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9.307692e+00</td>\n",
       "      <td>8.884615e+00</td>\n",
       "      <td>8.461538e+00</td>\n",
       "      <td>8.346154e+00</td>\n",
       "      <td>8.307692e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Start-up procedures to register a business (nu...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>3.203846e+01</td>\n",
       "      <td>3.157692e+01</td>\n",
       "      <td>2.928846e+01</td>\n",
       "      <td>2.782692e+01</td>\n",
       "      <td>2.682692e+01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Time required to start a business (days)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6.360000e+01</td>\n",
       "      <td>5.743636e+01</td>\n",
       "      <td>5.352273e+01</td>\n",
       "      <td>4.801364e+01</td>\n",
       "      <td>3.874545e+01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cost of business start-up procedures (% of GNI...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>3.150000e+00</td>\n",
       "      <td>3.100000e+00</td>\n",
       "      <td>3.075000e+00</td>\n",
       "      <td>3.025000e+00</td>\n",
       "      <td>3.025000e+00</td>\n",
       "      <td>3.050000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CPIA business regulatory environment rating (1...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>1.048446e+11</td>\n",
       "      <td>1.067829e+11</td>\n",
       "      <td>1.108089e+11</td>\n",
       "      <td>1.188674e+11</td>\n",
       "      <td>1.252819e+11</td>\n",
       "      <td>1.303555e+11</td>\n",
       "      <td>1.280260e+11</td>\n",
       "      <td>1.158040e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>7.607345e+11</td>\n",
       "      <td>7.617070e+11</td>\n",
       "      <td>7.793610e+11</td>\n",
       "      <td>8.023692e+11</td>\n",
       "      <td>8.280654e+11</td>\n",
       "      <td>8.206375e+11</td>\n",
       "      <td>8.527638e+11</td>\n",
       "      <td>GDP (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>1.087640e+03</td>\n",
       "      <td>1.085113e+03</td>\n",
       "      <td>1.102499e+03</td>\n",
       "      <td>1.157521e+03</td>\n",
       "      <td>1.193690e+03</td>\n",
       "      <td>1.214984e+03</td>\n",
       "      <td>1.167036e+03</td>\n",
       "      <td>1.032159e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>1.894322e+03</td>\n",
       "      <td>1.846333e+03</td>\n",
       "      <td>1.839113e+03</td>\n",
       "      <td>1.843555e+03</td>\n",
       "      <td>1.852862e+03</td>\n",
       "      <td>1.788647e+03</td>\n",
       "      <td>1.810928e+03</td>\n",
       "      <td>GDP per capita (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.142575e+10</td>\n",
       "      <td>1.254373e+10</td>\n",
       "      <td>1.345594e+10</td>\n",
       "      <td>1.428595e+10</td>\n",
       "      <td>1.492755e+10</td>\n",
       "      <td>1.444969e+10</td>\n",
       "      <td>...</td>\n",
       "      <td>8.053043e+11</td>\n",
       "      <td>7.352384e+11</td>\n",
       "      <td>6.829043e+11</td>\n",
       "      <td>7.015332e+11</td>\n",
       "      <td>7.460224e+11</td>\n",
       "      <td>7.577380e+11</td>\n",
       "      <td>8.141137e+11</td>\n",
       "      <td>GNI, Atlas method (current US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7.681818e+00</td>\n",
       "      <td>7.363636e+00</td>\n",
       "      <td>7.045455e+00</td>\n",
       "      <td>6.772727e+00</td>\n",
       "      <td>6.636364e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Start-up procedures to register a business (nu...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.517273e+01</td>\n",
       "      <td>2.420000e+01</td>\n",
       "      <td>1.790455e+01</td>\n",
       "      <td>1.774545e+01</td>\n",
       "      <td>1.525909e+01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Time required to start a business (days)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>3.878636e+01</td>\n",
       "      <td>3.828182e+01</td>\n",
       "      <td>3.361364e+01</td>\n",
       "      <td>3.456818e+01</td>\n",
       "      <td>2.779545e+01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cost of business start-up procedures (% of GNI...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.800000e+00</td>\n",
       "      <td>2.800000e+00</td>\n",
       "      <td>2.583333e+00</td>\n",
       "      <td>2.900000e+00</td>\n",
       "      <td>2.416667e+00</td>\n",
       "      <td>2.416667e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CPIA business regulatory environment rating (1...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.518701e+12</td>\n",
       "      <td>2.598673e+12</td>\n",
       "      <td>2.630964e+12</td>\n",
       "      <td>2.693157e+12</td>\n",
       "      <td>2.735879e+12</td>\n",
       "      <td>2.597680e+12</td>\n",
       "      <td>2.689385e+12</td>\n",
       "      <td>GDP (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6.359901e+03</td>\n",
       "      <td>6.431676e+03</td>\n",
       "      <td>6.386722e+03</td>\n",
       "      <td>6.414540e+03</td>\n",
       "      <td>6.394179e+03</td>\n",
       "      <td>5.956878e+03</td>\n",
       "      <td>6.050119e+03</td>\n",
       "      <td>GDP per capita (constant 2015 US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Weighted average</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.770420e+12</td>\n",
       "      <td>2.651921e+12</td>\n",
       "      <td>2.532787e+12</td>\n",
       "      <td>2.652001e+12</td>\n",
       "      <td>2.785018e+12</td>\n",
       "      <td>2.630381e+12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GNI, Atlas method (current US$)</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Gap-filled total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8.727273e+00</td>\n",
       "      <td>8.500000e+00</td>\n",
       "      <td>7.954545e+00</td>\n",
       "      <td>7.318182e+00</td>\n",
       "      <td>6.636364e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Start-up procedures to register a business (nu...</td>\n",
       "      <td>Annual</td>\n",
       "      <td>Unweighted average</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code     indicator_code          1960          1961  \\\n",
       "0              AFE  IC.REG.COST.PC.ZS           NaN           NaN   \n",
       "1              AFE     IQ.CPA.BREG.XQ           NaN           NaN   \n",
       "2              AFE     NY.GDP.MKTP.KD  1.538294e+11  1.541980e+11   \n",
       "3              AFE     NY.GDP.PCAP.KD  1.175735e+03  1.149361e+03   \n",
       "4              AFE     NY.GNP.ATLS.CD           NaN           NaN   \n",
       "5              AFE        IC.REG.PROC           NaN           NaN   \n",
       "6              AFE        IC.REG.DURS           NaN           NaN   \n",
       "7              AFW  IC.REG.COST.PC.ZS           NaN           NaN   \n",
       "8              AFW     IQ.CPA.BREG.XQ           NaN           NaN   \n",
       "9              AFW     NY.GDP.MKTP.KD  1.048446e+11  1.067829e+11   \n",
       "10             AFW     NY.GDP.PCAP.KD  1.087640e+03  1.085113e+03   \n",
       "11             AFW     NY.GNP.ATLS.CD           NaN           NaN   \n",
       "12             AFW        IC.REG.PROC           NaN           NaN   \n",
       "13             AFW        IC.REG.DURS           NaN           NaN   \n",
       "14             ARB  IC.REG.COST.PC.ZS           NaN           NaN   \n",
       "15             ARB     IQ.CPA.BREG.XQ           NaN           NaN   \n",
       "16             ARB     NY.GDP.MKTP.KD           NaN           NaN   \n",
       "17             ARB     NY.GDP.PCAP.KD           NaN           NaN   \n",
       "18             ARB     NY.GNP.ATLS.CD           NaN           NaN   \n",
       "19             ARB        IC.REG.PROC           NaN           NaN   \n",
       "\n",
       "            1962          1963          1964          1965          1966  \\\n",
       "0            NaN           NaN           NaN           NaN           NaN   \n",
       "1            NaN           NaN           NaN           NaN           NaN   \n",
       "2   1.665041e+11  1.751030e+11  1.831226e+11  1.928789e+11  2.004245e+11   \n",
       "3   1.209930e+03  1.240088e+03  1.263610e+03  1.296491e+03  1.312085e+03   \n",
       "4            NaN           NaN           NaN           NaN           NaN   \n",
       "5            NaN           NaN           NaN           NaN           NaN   \n",
       "6            NaN           NaN           NaN           NaN           NaN   \n",
       "7            NaN           NaN           NaN           NaN           NaN   \n",
       "8            NaN           NaN           NaN           NaN           NaN   \n",
       "9   1.108089e+11  1.188674e+11  1.252819e+11  1.303555e+11  1.280260e+11   \n",
       "10  1.102499e+03  1.157521e+03  1.193690e+03  1.214984e+03  1.167036e+03   \n",
       "11  1.142575e+10  1.254373e+10  1.345594e+10  1.428595e+10  1.492755e+10   \n",
       "12           NaN           NaN           NaN           NaN           NaN   \n",
       "13           NaN           NaN           NaN           NaN           NaN   \n",
       "14           NaN           NaN           NaN           NaN           NaN   \n",
       "15           NaN           NaN           NaN           NaN           NaN   \n",
       "16           NaN           NaN           NaN           NaN           NaN   \n",
       "17           NaN           NaN           NaN           NaN           NaN   \n",
       "18           NaN           NaN           NaN           NaN           NaN   \n",
       "19           NaN           NaN           NaN           NaN           NaN   \n",
       "\n",
       "            1967  ...          2015          2016          2017          2018  \\\n",
       "0            NaN  ...  5.509615e+01  5.563077e+01  5.107692e+01  4.173846e+01   \n",
       "1            NaN  ...  3.027778e+00  3.027778e+00  2.973684e+00  3.027778e+00   \n",
       "2   2.109604e+11  ...  9.242525e+11  9.447434e+11  9.689017e+11  9.929813e+11   \n",
       "3   1.344755e+03  ...  1.556316e+03  1.548813e+03  1.546796e+03  1.544078e+03   \n",
       "4            NaN  ...  9.476731e+11  9.003166e+11  9.142314e+11  9.421241e+11   \n",
       "5            NaN  ...  9.307692e+00  8.884615e+00  8.461538e+00  8.346154e+00   \n",
       "6            NaN  ...  3.203846e+01  3.157692e+01  2.928846e+01  2.782692e+01   \n",
       "7            NaN  ...  6.360000e+01  5.743636e+01  5.352273e+01  4.801364e+01   \n",
       "8            NaN  ...  3.150000e+00  3.100000e+00  3.075000e+00  3.025000e+00   \n",
       "9   1.158040e+11  ...  7.607345e+11  7.617070e+11  7.793610e+11  8.023692e+11   \n",
       "10  1.032159e+03  ...  1.894322e+03  1.846333e+03  1.839113e+03  1.843555e+03   \n",
       "11  1.444969e+10  ...  8.053043e+11  7.352384e+11  6.829043e+11  7.015332e+11   \n",
       "12           NaN  ...  7.681818e+00  7.363636e+00  7.045455e+00  6.772727e+00   \n",
       "13           NaN  ...  2.517273e+01  2.420000e+01  1.790455e+01  1.774545e+01   \n",
       "14           NaN  ...  3.878636e+01  3.828182e+01  3.361364e+01  3.456818e+01   \n",
       "15           NaN  ...  2.800000e+00  2.800000e+00  2.583333e+00  2.900000e+00   \n",
       "16           NaN  ...  2.518701e+12  2.598673e+12  2.630964e+12  2.693157e+12   \n",
       "17           NaN  ...  6.359901e+03  6.431676e+03  6.386722e+03  6.414540e+03   \n",
       "18           NaN  ...  2.770420e+12  2.651921e+12  2.532787e+12  2.652001e+12   \n",
       "19           NaN  ...  8.727273e+00  8.500000e+00  7.954545e+00  7.318182e+00   \n",
       "\n",
       "            2019          2020          2021  \\\n",
       "0   3.423077e+01           NaN           NaN   \n",
       "1   2.921053e+00  2.921053e+00           NaN   \n",
       "2   1.013099e+12  9.838518e+11  1.026194e+12   \n",
       "3   1.534890e+03  1.452730e+03  1.477249e+03   \n",
       "4   9.945128e+11  9.464769e+11  1.025115e+12   \n",
       "5   8.307692e+00           NaN           NaN   \n",
       "6   2.682692e+01           NaN           NaN   \n",
       "7   3.874545e+01           NaN           NaN   \n",
       "8   3.025000e+00  3.050000e+00           NaN   \n",
       "9   8.280654e+11  8.206375e+11  8.527638e+11   \n",
       "10  1.852862e+03  1.788647e+03  1.810928e+03   \n",
       "11  7.460224e+11  7.577380e+11  8.141137e+11   \n",
       "12  6.636364e+00           NaN           NaN   \n",
       "13  1.525909e+01           NaN           NaN   \n",
       "14  2.779545e+01           NaN           NaN   \n",
       "15  2.416667e+00  2.416667e+00           NaN   \n",
       "16  2.735879e+12  2.597680e+12  2.689385e+12   \n",
       "17  6.394179e+03  5.956878e+03  6.050119e+03   \n",
       "18  2.785018e+12  2.630381e+12           NaN   \n",
       "19  6.636364e+00           NaN           NaN   \n",
       "\n",
       "                                       indicator_name  periodicity  \\\n",
       "0   Cost of business start-up procedures (% of GNI...       Annual   \n",
       "1   CPIA business regulatory environment rating (1...       Annual   \n",
       "2                             GDP (constant 2015 US$)       Annual   \n",
       "3                  GDP per capita (constant 2015 US$)       Annual   \n",
       "4                     GNI, Atlas method (current US$)       Annual   \n",
       "5   Start-up procedures to register a business (nu...       Annual   \n",
       "6            Time required to start a business (days)       Annual   \n",
       "7   Cost of business start-up procedures (% of GNI...       Annual   \n",
       "8   CPIA business regulatory environment rating (1...       Annual   \n",
       "9                             GDP (constant 2015 US$)       Annual   \n",
       "10                 GDP per capita (constant 2015 US$)       Annual   \n",
       "11                    GNI, Atlas method (current US$)       Annual   \n",
       "12  Start-up procedures to register a business (nu...       Annual   \n",
       "13           Time required to start a business (days)       Annual   \n",
       "14  Cost of business start-up procedures (% of GNI...       Annual   \n",
       "15  CPIA business regulatory environment rating (1...       Annual   \n",
       "16                            GDP (constant 2015 US$)       Annual   \n",
       "17                 GDP per capita (constant 2015 US$)       Annual   \n",
       "18                    GNI, Atlas method (current US$)       Annual   \n",
       "19  Start-up procedures to register a business (nu...       Annual   \n",
       "\n",
       "    Aggregation_method  \n",
       "0   Unweighted average  \n",
       "1   Unweighted average  \n",
       "2     Gap-filled total  \n",
       "3     Weighted average  \n",
       "4     Gap-filled total  \n",
       "5   Unweighted average  \n",
       "6   Unweighted average  \n",
       "7   Unweighted average  \n",
       "8   Unweighted average  \n",
       "9     Gap-filled total  \n",
       "10    Weighted average  \n",
       "11    Gap-filled total  \n",
       "12  Unweighted average  \n",
       "13  Unweighted average  \n",
       "14  Unweighted average  \n",
       "15  Unweighted average  \n",
       "16    Gap-filled total  \n",
       "17    Weighted average  \n",
       "18    Gap-filled total  \n",
       "19  Unweighted average  \n",
       "\n",
       "[20 rows x 67 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "businessIndicatorsData = indicatorsData.join(businessIndicators, indicatorsData.indicator_code == businessIndicators.indicator_code).drop(businessIndicators.indicator_code)\n",
    "showDF(businessIndicatorsData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "c9afd186-4996-4d2f-aec6-71be15f9d1e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>34.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>2.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>1013098519257.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>1534.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>994512787038.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>8.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>26.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>38.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>3.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>828065395939.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>1852.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>746022427352.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>6.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>15.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>27.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>2.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>2735879327632.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>6394.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>2785018324949.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>6.64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code     indicator_code   indicator_value\n",
       "0              AFE  IC.REG.COST.PC.ZS             34.23\n",
       "1              AFE     IQ.CPA.BREG.XQ              2.92\n",
       "2              AFE     NY.GDP.MKTP.KD  1013098519257.66\n",
       "3              AFE     NY.GDP.PCAP.KD           1534.89\n",
       "4              AFE     NY.GNP.ATLS.CD   994512787038.08\n",
       "5              AFE        IC.REG.PROC              8.31\n",
       "6              AFE        IC.REG.DURS             26.83\n",
       "7              AFW  IC.REG.COST.PC.ZS             38.75\n",
       "8              AFW     IQ.CPA.BREG.XQ              3.03\n",
       "9              AFW     NY.GDP.MKTP.KD   828065395939.52\n",
       "10             AFW     NY.GDP.PCAP.KD           1852.86\n",
       "11             AFW     NY.GNP.ATLS.CD   746022427352.90\n",
       "12             AFW        IC.REG.PROC              6.64\n",
       "13             AFW        IC.REG.DURS             15.26\n",
       "14             ARB  IC.REG.COST.PC.ZS             27.80\n",
       "15             ARB     IQ.CPA.BREG.XQ              2.42\n",
       "16             ARB     NY.GDP.MKTP.KD  2735879327632.46\n",
       "17             ARB     NY.GDP.PCAP.KD           6394.18\n",
       "18             ARB     NY.GNP.ATLS.CD  2785018324949.32\n",
       "19             ARB        IC.REG.PROC              6.64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "recentIndicatorsData = businessIndicatorsData \\\n",
    "                        .select(\"wb_country_code\", \"indicator_code\",'2019') \\\n",
    "                        .withColumnRenamed(\"2019\", \"indicator_value\") \\\n",
    "                        .withColumn(\"indicator_value\", F.col(\"indicator_value\").cast(DecimalType(38, 2)))\n",
    "showDF(recentIndicatorsData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "ecd7a684-0258-43a2-8f8e-ba6f31e20d39",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [wb_country_code, cnt]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "businessIndicatorsCount = recentIndicatorsData.groupBy(\"wb_country_code\").agg(F.count(\"*\").alias(\"cnt\")).filter(\"cnt>9\")\n",
    "showDF(businessIndicatorsCount)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "15ec1687-d5ea-4398-9b4d-168c97f4fdc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>indicator_value</th>\n",
       "      <th>country_iso_code</th>\n",
       "      <th>wb_country_code</th>\n",
       "      <th>name</th>\n",
       "      <th>long_name</th>\n",
       "      <th>region</th>\n",
       "      <th>income_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>34.23</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>2.92</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>1013098519257.66</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>1534.89</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFE</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>994512787038.08</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>8.31</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AFE</td>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>26.83</td>\n",
       "      <td>ZH</td>\n",
       "      <td>AFE</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>Africa Eastern and Southern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>38.75</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>3.03</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>828065395939.52</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>1852.86</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AFW</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>746022427352.90</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>6.64</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AFW</td>\n",
       "      <td>IC.REG.DURS</td>\n",
       "      <td>15.26</td>\n",
       "      <td>ZI</td>\n",
       "      <td>AFW</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>Africa Western and Central</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IC.REG.COST.PC.ZS</td>\n",
       "      <td>27.80</td>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IQ.CPA.BREG.XQ</td>\n",
       "      <td>2.42</td>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GDP.MKTP.KD</td>\n",
       "      <td>2735879327632.46</td>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>6394.18</td>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ARB</td>\n",
       "      <td>NY.GNP.ATLS.CD</td>\n",
       "      <td>2785018324949.32</td>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ARB</td>\n",
       "      <td>IC.REG.PROC</td>\n",
       "      <td>6.64</td>\n",
       "      <td>1A</td>\n",
       "      <td>ARB</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>Arab World</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   wb_country_code     indicator_code   indicator_value country_iso_code  \\\n",
       "0              AFE  IC.REG.COST.PC.ZS             34.23               ZH   \n",
       "1              AFE     IQ.CPA.BREG.XQ              2.92               ZH   \n",
       "2              AFE     NY.GDP.MKTP.KD  1013098519257.66               ZH   \n",
       "3              AFE     NY.GDP.PCAP.KD           1534.89               ZH   \n",
       "4              AFE     NY.GNP.ATLS.CD   994512787038.08               ZH   \n",
       "5              AFE        IC.REG.PROC              8.31               ZH   \n",
       "6              AFE        IC.REG.DURS             26.83               ZH   \n",
       "7              AFW  IC.REG.COST.PC.ZS             38.75               ZI   \n",
       "8              AFW     IQ.CPA.BREG.XQ              3.03               ZI   \n",
       "9              AFW     NY.GDP.MKTP.KD   828065395939.52               ZI   \n",
       "10             AFW     NY.GDP.PCAP.KD           1852.86               ZI   \n",
       "11             AFW     NY.GNP.ATLS.CD   746022427352.90               ZI   \n",
       "12             AFW        IC.REG.PROC              6.64               ZI   \n",
       "13             AFW        IC.REG.DURS             15.26               ZI   \n",
       "14             ARB  IC.REG.COST.PC.ZS             27.80               1A   \n",
       "15             ARB     IQ.CPA.BREG.XQ              2.42               1A   \n",
       "16             ARB     NY.GDP.MKTP.KD  2735879327632.46               1A   \n",
       "17             ARB     NY.GDP.PCAP.KD           6394.18               1A   \n",
       "18             ARB     NY.GNP.ATLS.CD  2785018324949.32               1A   \n",
       "19             ARB        IC.REG.PROC              6.64               1A   \n",
       "\n",
       "   wb_country_code                         name                    long_name  \\\n",
       "0              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "1              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "2              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "3              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "4              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "5              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "6              AFE  Africa Eastern and Southern  Africa Eastern and Southern   \n",
       "7              AFW   Africa Western and Central   Africa Western and Central   \n",
       "8              AFW   Africa Western and Central   Africa Western and Central   \n",
       "9              AFW   Africa Western and Central   Africa Western and Central   \n",
       "10             AFW   Africa Western and Central   Africa Western and Central   \n",
       "11             AFW   Africa Western and Central   Africa Western and Central   \n",
       "12             AFW   Africa Western and Central   Africa Western and Central   \n",
       "13             AFW   Africa Western and Central   Africa Western and Central   \n",
       "14             ARB                   Arab World                   Arab World   \n",
       "15             ARB                   Arab World                   Arab World   \n",
       "16             ARB                   Arab World                   Arab World   \n",
       "17             ARB                   Arab World                   Arab World   \n",
       "18             ARB                   Arab World                   Arab World   \n",
       "19             ARB                   Arab World                   Arab World   \n",
       "\n",
       "   region income_group  \n",
       "0    None         None  \n",
       "1    None         None  \n",
       "2    None         None  \n",
       "3    None         None  \n",
       "4    None         None  \n",
       "5    None         None  \n",
       "6    None         None  \n",
       "7    None         None  \n",
       "8    None         None  \n",
       "9    None         None  \n",
       "10   None         None  \n",
       "11   None         None  \n",
       "12   None         None  \n",
       "13   None         None  \n",
       "14   None         None  \n",
       "15   None         None  \n",
       "16   None         None  \n",
       "17   None         None  \n",
       "18   None         None  \n",
       "19   None         None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "countryBusinessIndicator = recentIndicatorsData.join(countryDimFinal, recentIndicatorsData.wb_country_code == countryDimFinal.wb_country_code)\n",
    "showDF(countryBusinessIndicator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "d79b1273-be54-4725-96d0-7e212f4484f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country ISO Code</th>\n",
       "      <th>Country Name</th>\n",
       "      <th>Startup Cost Pct of GNI</th>\n",
       "      <th>Startup Time</th>\n",
       "      <th>Startup Procedures</th>\n",
       "      <th>Business Regulation</th>\n",
       "      <th>GDP</th>\n",
       "      <th>GDP per capita</th>\n",
       "      <th>GNI</th>\n",
       "      <th>Startup Cost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BJ</td>\n",
       "      <td>Benin</td>\n",
       "      <td>3.50</td>\n",
       "      <td>8.50</td>\n",
       "      <td>6.00</td>\n",
       "      <td>3.50</td>\n",
       "      <td>14179807326.72</td>\n",
       "      <td>1201.56</td>\n",
       "      <td>14803390762.06</td>\n",
       "      <td>518118676.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>XC</td>\n",
       "      <td>Euro area</td>\n",
       "      <td>3.21</td>\n",
       "      <td>9.82</td>\n",
       "      <td>5.05</td>\n",
       "      <td>None</td>\n",
       "      <td>12623114142874.70</td>\n",
       "      <td>36879.14</td>\n",
       "      <td>13798162524579.10</td>\n",
       "      <td>442921017038.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LY</td>\n",
       "      <td>Libya</td>\n",
       "      <td>24.60</td>\n",
       "      <td>35.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>None</td>\n",
       "      <td>60950218438.49</td>\n",
       "      <td>8993.09</td>\n",
       "      <td>69161249722.12</td>\n",
       "      <td>17013667431.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KZ</td>\n",
       "      <td>Kazakhstan</td>\n",
       "      <td>0.20</td>\n",
       "      <td>5.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>None</td>\n",
       "      <td>211106972039.01</td>\n",
       "      <td>11402.76</td>\n",
       "      <td>163361654737.11</td>\n",
       "      <td>326723309.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JM</td>\n",
       "      <td>Jamaica</td>\n",
       "      <td>4.20</td>\n",
       "      <td>3.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>None</td>\n",
       "      <td>14934128282.74</td>\n",
       "      <td>5065.37</td>\n",
       "      <td>15466765494.66</td>\n",
       "      <td>649604150.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NO</td>\n",
       "      <td>Norway</td>\n",
       "      <td>0.80</td>\n",
       "      <td>4.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>None</td>\n",
       "      <td>406468037614.96</td>\n",
       "      <td>76005.22</td>\n",
       "      <td>436746877155.38</td>\n",
       "      <td>3493975017.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AG</td>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>8.00</td>\n",
       "      <td>19.00</td>\n",
       "      <td>9.00</td>\n",
       "      <td>None</td>\n",
       "      <td>1630215842.30</td>\n",
       "      <td>16786.45</td>\n",
       "      <td>1594604632.72</td>\n",
       "      <td>127568370.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>IR</td>\n",
       "      <td>Iran</td>\n",
       "      <td>1.10</td>\n",
       "      <td>72.50</td>\n",
       "      <td>10.00</td>\n",
       "      <td>None</td>\n",
       "      <td>440183212449.35</td>\n",
       "      <td>5308.92</td>\n",
       "      <td>353025324793.28</td>\n",
       "      <td>3883278572.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CG</td>\n",
       "      <td>Congo</td>\n",
       "      <td>62.20</td>\n",
       "      <td>49.50</td>\n",
       "      <td>11.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>9647394773.66</td>\n",
       "      <td>1793.03</td>\n",
       "      <td>9746848755.23</td>\n",
       "      <td>6062539925.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>EU</td>\n",
       "      <td>European Union</td>\n",
       "      <td>3.20</td>\n",
       "      <td>12.17</td>\n",
       "      <td>5.37</td>\n",
       "      <td>None</td>\n",
       "      <td>14772269194026.80</td>\n",
       "      <td>33032.96</td>\n",
       "      <td>16101472914830.00</td>\n",
       "      <td>515247133274.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AR</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>5.00</td>\n",
       "      <td>11.50</td>\n",
       "      <td>12.00</td>\n",
       "      <td>None</td>\n",
       "      <td>571304530654.92</td>\n",
       "      <td>12712.97</td>\n",
       "      <td>506161996612.64</td>\n",
       "      <td>25308099830.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>HT</td>\n",
       "      <td>Haiti</td>\n",
       "      <td>179.70</td>\n",
       "      <td>97.00</td>\n",
       "      <td>12.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>15474153940.15</td>\n",
       "      <td>1373.88</td>\n",
       "      <td>15626724070.54</td>\n",
       "      <td>28081223154.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>QA</td>\n",
       "      <td>Qatar</td>\n",
       "      <td>6.30</td>\n",
       "      <td>8.70</td>\n",
       "      <td>8.00</td>\n",
       "      <td>None</td>\n",
       "      <td>167515127917.18</td>\n",
       "      <td>59149.34</td>\n",
       "      <td>173269092486.30</td>\n",
       "      <td>10915952826.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>XE</td>\n",
       "      <td>Heavily indebted poor countries (HIPC)</td>\n",
       "      <td>44.37</td>\n",
       "      <td>22.54</td>\n",
       "      <td>7.26</td>\n",
       "      <td>2.86</td>\n",
       "      <td>821855972179.45</td>\n",
       "      <td>1025.13</td>\n",
       "      <td>795744512417.00</td>\n",
       "      <td>353071840159.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>BB</td>\n",
       "      <td>Barbados</td>\n",
       "      <td>7.30</td>\n",
       "      <td>16.00</td>\n",
       "      <td>7.00</td>\n",
       "      <td>None</td>\n",
       "      <td>4787093962.03</td>\n",
       "      <td>16678.55</td>\n",
       "      <td>4915707228.49</td>\n",
       "      <td>358846627.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>LT</td>\n",
       "      <td>Lithuania</td>\n",
       "      <td>0.50</td>\n",
       "      <td>5.50</td>\n",
       "      <td>4.00</td>\n",
       "      <td>None</td>\n",
       "      <td>48174429493.23</td>\n",
       "      <td>17241.26</td>\n",
       "      <td>53267968808.46</td>\n",
       "      <td>266339844.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>S4</td>\n",
       "      <td>Other small states</td>\n",
       "      <td>15.83</td>\n",
       "      <td>15.58</td>\n",
       "      <td>7.23</td>\n",
       "      <td>3.05</td>\n",
       "      <td>403195694802.60</td>\n",
       "      <td>12856.51</td>\n",
       "      <td>426308417509.05</td>\n",
       "      <td>67484622491.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>SL</td>\n",
       "      <td>Sierra Leone</td>\n",
       "      <td>7.60</td>\n",
       "      <td>8.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>3.00</td>\n",
       "      <td>5076711739.04</td>\n",
       "      <td>649.76</td>\n",
       "      <td>4158883999.26</td>\n",
       "      <td>316075183.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>OM</td>\n",
       "      <td>Oman</td>\n",
       "      <td>3.10</td>\n",
       "      <td>4.30</td>\n",
       "      <td>4.00</td>\n",
       "      <td>None</td>\n",
       "      <td>83053174485.83</td>\n",
       "      <td>16694.13</td>\n",
       "      <td>81743368604.87</td>\n",
       "      <td>2534044426.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>PK</td>\n",
       "      <td>Pakistan</td>\n",
       "      <td>6.70</td>\n",
       "      <td>16.50</td>\n",
       "      <td>5.00</td>\n",
       "      <td>3.50</td>\n",
       "      <td>324411993155.44</td>\n",
       "      <td>1497.99</td>\n",
       "      <td>349836038441.43</td>\n",
       "      <td>23439014575.58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Country ISO Code                            Country Name  \\\n",
       "0                BJ                                   Benin   \n",
       "1                XC                               Euro area   \n",
       "2                LY                                   Libya   \n",
       "3                KZ                              Kazakhstan   \n",
       "4                JM                                 Jamaica   \n",
       "5                NO                                  Norway   \n",
       "6                AG                     Antigua and Barbuda   \n",
       "7                IR                                    Iran   \n",
       "8                CG                                   Congo   \n",
       "9                EU                          European Union   \n",
       "10               AR                               Argentina   \n",
       "11               HT                                   Haiti   \n",
       "12               QA                                   Qatar   \n",
       "13               XE  Heavily indebted poor countries (HIPC)   \n",
       "14               BB                                Barbados   \n",
       "15               LT                               Lithuania   \n",
       "16               S4                      Other small states   \n",
       "17               SL                            Sierra Leone   \n",
       "18               OM                                    Oman   \n",
       "19               PK                                Pakistan   \n",
       "\n",
       "   Startup Cost Pct of GNI Startup Time Startup Procedures  \\\n",
       "0                     3.50         8.50               6.00   \n",
       "1                     3.21         9.82               5.05   \n",
       "2                    24.60        35.00              10.00   \n",
       "3                     0.20         5.00               4.00   \n",
       "4                     4.20         3.00               2.00   \n",
       "5                     0.80         4.00               4.00   \n",
       "6                     8.00        19.00               9.00   \n",
       "7                     1.10        72.50              10.00   \n",
       "8                    62.20        49.50              11.00   \n",
       "9                     3.20        12.17               5.37   \n",
       "10                    5.00        11.50              12.00   \n",
       "11                  179.70        97.00              12.00   \n",
       "12                    6.30         8.70               8.00   \n",
       "13                   44.37        22.54               7.26   \n",
       "14                    7.30        16.00               7.00   \n",
       "15                    0.50         5.50               4.00   \n",
       "16                   15.83        15.58               7.23   \n",
       "17                    7.60         8.00               5.00   \n",
       "18                    3.10         4.30               4.00   \n",
       "19                    6.70        16.50               5.00   \n",
       "\n",
       "   Business Regulation                GDP GDP per capita                GNI  \\\n",
       "0                 3.50     14179807326.72        1201.56     14803390762.06   \n",
       "1                 None  12623114142874.70       36879.14  13798162524579.10   \n",
       "2                 None     60950218438.49        8993.09     69161249722.12   \n",
       "3                 None    211106972039.01       11402.76    163361654737.11   \n",
       "4                 None     14934128282.74        5065.37     15466765494.66   \n",
       "5                 None    406468037614.96       76005.22    436746877155.38   \n",
       "6                 None      1630215842.30       16786.45      1594604632.72   \n",
       "7                 None    440183212449.35        5308.92    353025324793.28   \n",
       "8                 2.00      9647394773.66        1793.03      9746848755.23   \n",
       "9                 None  14772269194026.80       33032.96  16101472914830.00   \n",
       "10                None    571304530654.92       12712.97    506161996612.64   \n",
       "11                2.00     15474153940.15        1373.88     15626724070.54   \n",
       "12                None    167515127917.18       59149.34    173269092486.30   \n",
       "13                2.86    821855972179.45        1025.13    795744512417.00   \n",
       "14                None      4787093962.03       16678.55      4915707228.49   \n",
       "15                None     48174429493.23       17241.26     53267968808.46   \n",
       "16                3.05    403195694802.60       12856.51    426308417509.05   \n",
       "17                3.00      5076711739.04         649.76      4158883999.26   \n",
       "18                None     83053174485.83       16694.13     81743368604.87   \n",
       "19                3.50    324411993155.44        1497.99    349836038441.43   \n",
       "\n",
       "       Startup Cost  \n",
       "0      518118676.67  \n",
       "1   442921017038.99  \n",
       "2    17013667431.64  \n",
       "3      326723309.47  \n",
       "4      649604150.78  \n",
       "5     3493975017.24  \n",
       "6      127568370.62  \n",
       "7     3883278572.73  \n",
       "8     6062539925.75  \n",
       "9   515247133274.56  \n",
       "10   25308099830.63  \n",
       "11   28081223154.76  \n",
       "12   10915952826.64  \n",
       "13  353071840159.42  \n",
       "14     358846627.68  \n",
       "15     266339844.04  \n",
       "16   67484622491.68  \n",
       "17     316075183.94  \n",
       "18    2534044426.75  \n",
       "19   23439014575.58  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "countryBusinessStartupPivot = countryBusinessIndicator \\\n",
    "                                .select(\"country_iso_code\" \\\n",
    "                                        ,\"name\" \\\n",
    "                                        ,\"indicator_code\" \\\n",
    "                                        ,\"indicator_value\") \\\n",
    "                                .groupBy(\"country_iso_code\",\"name\").pivot(\"indicator_code\").sum(\"indicator_value\") \\\n",
    "                                .withColumnRenamed(\"country_iso_code\", \"Country ISO Code\") \\\n",
    "                                .withColumnRenamed(\"name\", \"Country Name\") \\\n",
    "                                .withColumnRenamed(\"IC.REG.COST.PC.ZS\", \"Startup Cost Pct of GNI\") \\\n",
    "                                .withColumnRenamed(\"IC.REG.DURS\", \"Startup Time\") \\\n",
    "                                .withColumnRenamed(\"IC.REG.PROC\",\"Startup Procedures\") \\\n",
    "                                .withColumnRenamed(\"IQ.CPA.BREG.XQ\",\"Business Regulation\") \\\n",
    "                                .withColumnRenamed(\"NY.GDP.MKTP.KD\", \"GDP\") \\\n",
    "                                .withColumnRenamed(\"NY.GDP.PCAP.KD\", \"GDP per capita\") \\\n",
    "                                .withColumnRenamed(\"NY.GNP.ATLS.CD\",\"GNI\") \\\n",
    "                                .withColumn('Startup Cost', (F.col('GNI') * F.col('Startup Cost Pct of GNI') / F.lit(100.0)).cast(DecimalType(38, 2))) \\\n",
    "                                .filter(F.col('GNI') > 0) \\\n",
    "                                .filter(F.col(\"Startup Cost\").isNotNull())\n",
    "                                \n",
    "showDF(countryBusinessStartupPivot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "26b8ad69-0d02-4e2d-8982-afbc9920f304",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#write the business startup data to an output csv file\n",
    "countryBusinessStartupPivot \\\n",
    "    .select(\"Country ISO Code\", \"Country Name\", \"GDP\", \"GDP Per Capita\", \"GNI\", \\\n",
    "            \"Startup Cost\", \"Startup Cost Pct of GNI\", \"Startup Time\", \"Startup Procedures\", \\\n",
    "            \"Business Regulation\") \\\n",
    "    .coalesce(1) \\\n",
    "    .write.csv('output/BusinessStartupData', mode='overwrite', header='true')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}