adding Data filter

3_Streamlit_dashboard/3_STREAMlit_dashboard.py

@@ -17,6 +17,13 @@
 import bokeh.models as bmo
 from bokeh.layouts import column
 
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
 def bokeh_scatter(data_slice, data_category, data_category_y):
     # Create a ColumnDataSource
     # Color for no risk
@@ -149,7 +156,80 @@ def plot_histogram(data, id, age, category):
 
     st.pyplot(fig)
 
+def filter_dataframe(df: pd.DataFrame):
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
 
+    Args:
+        df (pd.DataFrame): Original dataframe
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+    modify = st.checkbox("Add filters")
+
+    if not modify:
+        return df
+
+    df = df.copy()
+
+    # Try to convert datetimes into a standard format (datetime, no timezone)
+    for col in df.columns:
+        if is_object_dtype(df[col]):
+            try:
+                df[col] = pd.to_datetime(df[col])
+            except Exception:
+                pass
+
+        if is_datetime64_any_dtype(df[col]):
+            df[col] = df[col].dt.tz_localize(None)
+
+    modification_container = st.container()
+
+    with modification_container:
+        to_filter_columns = st.multiselect("Filter dataframe on", df.columns)
+        for column in to_filter_columns:
+            left, right = st.columns((1, 20))
+            left.write("↳")
+            # Treat columns with < 10 unique values as categorical
+            if is_categorical_dtype(df[column]) or df[column].nunique() < 10:
+                user_cat_input = right.multiselect(
+                    f"Values for {column}",
+                    df[column].unique(),
+                    default=list(df[column].unique()),
+                )
+                df = df[df[column].isin(user_cat_input)]
+            elif is_numeric_dtype(df[column]):
+                _min = float(df[column].min())
+                _max = float(df[column].max())
+                step = (_max - _min) / 100
+                user_num_input = right.slider(
+                    f"Values for {column}",
+                    _min,
+                    _max,
+                    (_min, _max),
+                    step=step,
+                )
+                df = df[df[column].between(*user_num_input)]
+            elif is_datetime64_any_dtype(df[column]):
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(
+                        df[column].min(),
+                        df[column].max(),
+                    ),
+                )
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df = df.loc[df[column].between(start_date, end_date)]
+            else:
+                user_text_input = right.text_input(
+                    f"Substring or regex in {column}",
+                )
+                if user_text_input:
+                    df = df[df[column].str.contains(user_text_input)]
+    return df
 
 def main():
     MLFLOW_URI = 'https://fastapi-cd-webapp.azurewebsites.net/predict'
@@ -161,7 +241,6 @@ def main():
         MLFLOW_URI = MLFLOW_URI
 
 
-
     ids_test = pd.read_csv('data/test_ids.csv')
     id_list = ids_test.iloc[:,0].values.tolist()
 
@@ -246,18 +325,32 @@ def main():
     elif selected == "Dashboard":
         c1, c2= st.columns(2)
         with st.container():
-            c1.write("X")
-            c2.write("Y")
+            c1.write("")
+            c2.write("")
+        with c1:
+            selected_x = st.selectbox('Select X', options=columns, index=1, format_func=lambda x: x if x else 'Search...')
+        with c2:
+            selected_y = st.selectbox('Select Y', options=columns, index=3, format_func=lambda x: x if x else 'Search...')
+        #st.write(data_slice)
+
+        data_filter = filter_dataframe(data_slice)
+        st.write(data_slice[['age','Work_in_years','AMT_INCOME_TOTAL','AMT_CREDIT','CODE_GENDER']].describe())
+        st.write(data_filter[['age','Work_in_years','AMT_INCOME_TOTAL','AMT_CREDIT','CODE_GENDER']].describe())
+
+        c1, c2, c3 = st.columns(3)
+        with st.container():
+            c1.write("")
+            c2.write("")
+            c3.write("")
         with c1:
-            selected_y = st.selectbox('Select Y', options=columns, index=1, format_func=lambda x: x if x else 'Search...')
+            st.write(data_filter[['TARGET']].value_counts())
         with c2:
-            selected_x = st.selectbox('Select X', options=columns, index=3, format_func=lambda x: x if x else 'Search...')
-        st.write(data_slice)
-        chart_data = pd.DataFrame(data_slice, columns=[selected_y, selected_x])
-        st.write(chart_data)
-        st.scatter_chart(chart_data, x=selected_x, y= selected_y,)
+            st.write(data_filter[['CODE_GENDER']].value_counts())
+        with c3:
+            st.write(data_filter[['CODE_GENDER','TARGET']].value_counts())
 
-        #bokeh_scatter(chart_data[selected_x], chart_data[selected_y], chart_data['TARGET'])
+        chart_data = pd.DataFrame(data_filter, columns=[selected_y, selected_x])
+        st.scatter_chart(chart_data, x=selected_x, y= selected_y,)
 
     elif selected == "Scatter Plot":
         st.write("bokeh scatter here")