[docs]
-
def hosmer_lemeshow_test(reliability, confidence, bin_count, df=None):
+
def hosmer_lemeshow_test(reliability, confidence, bin_count, df=None, **kwargs):
"""
Compute the Hosmer-Lemeshow test for goodness of fit.
@@ -667,7 +667,7 @@
Source code for calzone.metrics
results["MCE-H"] = mce_h_class
elif metric == "HL-H":
hl_h_score, hl_h, _ = hosmer_lemeshow_test(
- acc_H_class, confidence_H_class, bin_count_H_class
+ acc_H_class, confidence_H_class, bin_count_H_class, **kwargs
)
results["HL-H score"] = hl_h_score
results["HL-H p-value"] = hl_h
@@ -712,7 +712,7 @@ Source code for calzone.metrics
results["MCE-C"] = mce_c_class
elif metric == "HL-C":
hl_c_score, hl_c, _ = hosmer_lemeshow_test(
- acc_C_class, confidence_C_class, bin_count_C_class
+ acc_C_class, confidence_C_class, bin_count_C_class, **kwargs
)
results["HL-C score"] = hl_c_score
results["HL-C p-value"] = hl_c
diff --git a/docs/build/html/_sources/notebooks/hl_test.ipynb.txt b/docs/build/html/_sources/notebooks/hl_test.ipynb.txt
index ec53edd..41e8204 100644
--- a/docs/build/html/_sources/notebooks/hl_test.ipynb.txt
+++ b/docs/build/html/_sources/notebooks/hl_test.ipynb.txt
@@ -20,7 +20,7 @@
"\\text{HL} = \\sum_{m=1}^{M} \\left[\\frac{(O_{1,m}-E_{1,m})^2}{E_{1,m}} + \\frac{(O_{0,m}-E_{0,m})^2}{E_{0,m}}\\right] = \\sum_{m=1}^{M} \\frac{(O_{1,m}-E_{1,m})^2}{E_{1,m}(1-\\frac{E_{1,m}}{N_m})} \\sim \\chi^2_{M-2}\n",
"$$\n",
"\n",
- "where $E_{1,m}$ is the expected number of class 1 events in the $\\text{m}^{th}$ bin, $O_{1,m}$ is the observed number of class 1 events in the $\\text{m}^{th}$ bin, $N_m$ is the total number of observations in the $\\text{m}^{th}$ bin, and $M$ is the number of bins. The HL test statistic is distributed as a chi-squared distribution with $M-2$ degrees of freedom. We can then use this test statistic to calculate the p-value for the test and determine whether we can reject the null hypothesis that the model is well-calibrated. (Notice that the degree of freedom of HL test is $M-2$ by default but some literature suggests that the degree of freedom should be $M$ instead when the samples is not used for training. We could not find a proof to this statement and we provides the option to specify the degree of freedom in the `calzone`. The default value is still $M-2$.)"
+ "where $E_{1,m}$ is the expected number of class 1 events in the $\\text{m}^{th}$ bin, $O_{1,m}$ is the observed number of class 1 events in the $\\text{m}^{th}$ bin, $N_m$ is the total number of observations in the $\\text{m}^{th}$ bin, and $M$ is the number of bins. The HL test statistic is distributed as a chi-squared distribution with $M-2$ degrees of freedom. We can then use this test statistic to calculate the p-value for the test and determine whether we can reject the null hypothesis that the model is well-calibrated. Notice that the degree of freedom of HL test is $M-2$ by default but some literature suggests that the degree of freedom should be $M$ instead when the samples is not used for training. We provides the option to specify the degree of freedom in the `calzone`. The default value is still $M-2$."
]
},
{
@@ -280,7 +280,54 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "We can see that both the equal-width and the equal-count method have the incorrect size."
+ "We can see that both the equal-width and the equal-count method have the incorrect size. The simulation support the claim that the degree of freedom should be M instead of M-2. We can show it with simulation. We are not proving the claim here since it is beyond the scope of this documentation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The size of HL-H with df=M is : 0.047\n",
+ "The size of HL-C with df=M is : 0.055\n"
+ ]
+ }
+ ],
+ "source": [
+ "### The size of HL Test\n",
+ "from calzone.utils import fake_binary_data_generator\n",
+ "from importlib import reload\n",
+ "import calzone.metrics\n",
+ "reload(calzone.metrics)\n",
+ "from calzone.metrics import CalibrationMetrics\n",
+ "np.random.seed(123)\n",
+ "fakedata_generator = fake_binary_data_generator(alpha_val=0.5, beta_val=0.5)\n",
+ "cal_metrics = CalibrationMetrics()\n",
+ "sample_size = 1000\n",
+ "simulation_size = 10000\n",
+ "results = []\n",
+ "# generate data\n",
+ "for i in range(simulation_size):\n",
+ " X, y = fakedata_generator.generate_data(sample_size)\n",
+ " if i == 0:\n",
+ " tempresult = cal_metrics.calculate_metrics(y, X, ['HL-H', 'HL-C'],return_numpy=False, df = 10)\n",
+ " keys = list(tempresult.keys())\n",
+ " results.append(np.array(list(tempresult.values())))\n",
+ " else:\n",
+ " tempresult = cal_metrics.calculate_metrics(y, X, ['HL-H', 'HL-C'],return_numpy=True, df = 10)\n",
+ " results.append(tempresult)\n",
+ "results = np.array(results)\n",
+ "\n",
+ "hl_h_pvalue = results[:,1]\n",
+ "hl_c_pvalue = results[:,3]\n",
+ "size_h = np.mean(hl_h_pvalue < 0.05)\n",
+ "size_c = np.mean(hl_c_pvalue < 0.05)\n",
+ "print(\"The size of HL-H with df=M is :\", round(size_h,3))\n",
+ "print(\"The size of HL-C with df=M is :\", round(size_c,3))"
]
},
{
diff --git a/docs/build/html/_sources/notebooks/quickstart.ipynb.txt b/docs/build/html/_sources/notebooks/quickstart.ipynb.txt
index 1e5b953..63d7bc1 100644
--- a/docs/build/html/_sources/notebooks/quickstart.ipynb.txt
+++ b/docs/build/html/_sources/notebooks/quickstart.ipynb.txt
@@ -184,7 +184,8 @@
" [--prevalence_adjustment] [--n_bootstrap N_BOOTSTRAP]\n",
" [--bootstrap_ci BOOTSTRAP_CI]\n",
" [--class_to_calculate CLASS_TO_CALCULATE]\n",
- " [--num_bins NUM_BINS] [--topclass]\n",
+ " [--num_bins NUM_BINS]\n",
+ " [--hl_test_validation HL_TEST_VALIDATION] [--topclass]\n",
" [--save_metrics SAVE_METRICS] [--plot]\n",
" [--plot_bins PLOT_BINS] [--save_plot SAVE_PLOT]\n",
" [--save_diagram_output SAVE_DIAGRAM_OUTPUT] [--verbose]\n",
@@ -211,6 +212,9 @@
" Class to calculate metrics for (default: 1)\n",
" --num_bins NUM_BINS Number of bins for ECE/MCE/HL calculations (default:\n",
" 10)\n",
+ " --hl_test_validation HL_TEST_VALIDATION\n",
+ " Using nbin instead of nbin-2 as HL test DOF. Use it if\n",
+ " the dataset is validation set.\n",
" --topclass Whether to transform the problem to top-class problem.\n",
" --save_metrics SAVE_METRICS\n",
" Save the metrics to a csv file\n",
@@ -218,7 +222,8 @@
" --plot_bins PLOT_BINS\n",
" Number of bins for reliability diagram\n",
" --save_plot SAVE_PLOT\n",
- " Save the plot to a file\n",
+ " Save the plot to a file. Must end with valid image\n",
+ " formats.\n",
" --save_diagram_output SAVE_DIAGRAM_OUTPUT\n",
" Save the reliability diagram output to a file\n",
" --verbose Print verbose output\n"
@@ -290,7 +295,8 @@
"--verbose \\\n",
"--save_diagram_output '../../../example_data/simulated_welldata_diagram_output.csv' \n",
"### save_diagram_output only when you want to save the reliability diagram output\n",
- "#--prevalence_adjustment # only when you want to apply prevalence adjustment"
+ "#--prevalence_adjustment # only when you want to apply prevalence adjustment\n",
+ "#--hl_test_validation #use it only when the data is from validation set"
]
},
{
diff --git a/docs/build/html/calzone.html b/docs/build/html/calzone.html
index 39c487b..707b549 100644
--- a/docs/build/html/calzone.html
+++ b/docs/build/html/calzone.html
@@ -386,7 +386,7 @@ Submodules[source]
Compute the Hosmer-Lemeshow test for goodness of fit.
This test is used to assess the calibration of binary classification models with full probability outputs.
It compares observed and expected frequencies of events in groups of the data.
diff --git a/docs/build/html/notebooks/hl_test.html b/docs/build/html/notebooks/hl_test.html
index 6f302f1..54ade3e 100644
--- a/docs/build/html/notebooks/hl_test.html
+++ b/docs/build/html/notebooks/hl_test.html
@@ -108,7 +108,7 @@ Theoretical Background
where \(E_{1,m}\) is the expected number of class 1 events in the \(\text{m}^{th}\) bin, \(O_{1,m}\) is the observed number of class 1 events in the \(\text{m}^{th}\) bin, \(N_m\) is the total number of observations in the \(\text{m}^{th}\) bin, and \(M\) is the number of bins. The HL test statistic is distributed as a chi-squared distribution with \(M-2\) degrees of freedom. We can then use this test statistic to calculate the p-value for the test and determine
-whether we can reject the null hypothesis that the model is well-calibrated. (Notice that the degree of freedom of HL test is \(M-2\) by default but some literature suggests that the degree of freedom should be \(M\) instead when the samples is not used for training. We could not find a proof to this statement and we provides the option to specify the degree of freedom in the calzone. The default value is still \(M-2\).)
+whether we can reject the null hypothesis that the model is well-calibrated. Notice that the degree of freedom of HL test is \(M-2\) by default but some literature suggests that the degree of freedom should be \(M\) instead when the samples is not used for training. We provides the option to specify the degree of freedom in the calzone. The default value is still \(M-2\).
Pros of HL test
@@ -310,7 +310,53 @@ Size of HL test
-We can see that both the equal-width and the equal-count method have the incorrect size.
+We can see that both the equal-width and the equal-count method have the incorrect size. The simulation support the claim that the degree of freedom should be M instead of M-2. We can show it with simulation. We are not proving the claim here since it is beyond the scope of this documentation.
+
+
+
+
+
+
+The size of HL-H with df=M is : 0.047
+The size of HL-C with df=M is : 0.055
+
+
Reference
diff --git a/docs/build/html/notebooks/hl_test.ipynb b/docs/build/html/notebooks/hl_test.ipynb
index ec53edd..41e8204 100644
--- a/docs/build/html/notebooks/hl_test.ipynb
+++ b/docs/build/html/notebooks/hl_test.ipynb
@@ -20,7 +20,7 @@
"\\text{HL} = \\sum_{m=1}^{M} \\left[\\frac{(O_{1,m}-E_{1,m})^2}{E_{1,m}} + \\frac{(O_{0,m}-E_{0,m})^2}{E_{0,m}}\\right] = \\sum_{m=1}^{M} \\frac{(O_{1,m}-E_{1,m})^2}{E_{1,m}(1-\\frac{E_{1,m}}{N_m})} \\sim \\chi^2_{M-2}\n",
"$$\n",
"\n",
- "where $E_{1,m}$ is the expected number of class 1 events in the $\\text{m}^{th}$ bin, $O_{1,m}$ is the observed number of class 1 events in the $\\text{m}^{th}$ bin, $N_m$ is the total number of observations in the $\\text{m}^{th}$ bin, and $M$ is the number of bins. The HL test statistic is distributed as a chi-squared distribution with $M-2$ degrees of freedom. We can then use this test statistic to calculate the p-value for the test and determine whether we can reject the null hypothesis that the model is well-calibrated. (Notice that the degree of freedom of HL test is $M-2$ by default but some literature suggests that the degree of freedom should be $M$ instead when the samples is not used for training. We could not find a proof to this statement and we provides the option to specify the degree of freedom in the `calzone`. The default value is still $M-2$.)"
+ "where $E_{1,m}$ is the expected number of class 1 events in the $\\text{m}^{th}$ bin, $O_{1,m}$ is the observed number of class 1 events in the $\\text{m}^{th}$ bin, $N_m$ is the total number of observations in the $\\text{m}^{th}$ bin, and $M$ is the number of bins. The HL test statistic is distributed as a chi-squared distribution with $M-2$ degrees of freedom. We can then use this test statistic to calculate the p-value for the test and determine whether we can reject the null hypothesis that the model is well-calibrated. Notice that the degree of freedom of HL test is $M-2$ by default but some literature suggests that the degree of freedom should be $M$ instead when the samples is not used for training. We provides the option to specify the degree of freedom in the `calzone`. The default value is still $M-2$."
]
},
{
@@ -280,7 +280,54 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "We can see that both the equal-width and the equal-count method have the incorrect size."
+ "We can see that both the equal-width and the equal-count method have the incorrect size. The simulation support the claim that the degree of freedom should be M instead of M-2. We can show it with simulation. We are not proving the claim here since it is beyond the scope of this documentation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The size of HL-H with df=M is : 0.047\n",
+ "The size of HL-C with df=M is : 0.055\n"
+ ]
+ }
+ ],
+ "source": [
+ "### The size of HL Test\n",
+ "from calzone.utils import fake_binary_data_generator\n",
+ "from importlib import reload\n",
+ "import calzone.metrics\n",
+ "reload(calzone.metrics)\n",
+ "from calzone.metrics import CalibrationMetrics\n",
+ "np.random.seed(123)\n",
+ "fakedata_generator = fake_binary_data_generator(alpha_val=0.5, beta_val=0.5)\n",
+ "cal_metrics = CalibrationMetrics()\n",
+ "sample_size = 1000\n",
+ "simulation_size = 10000\n",
+ "results = []\n",
+ "# generate data\n",
+ "for i in range(simulation_size):\n",
+ " X, y = fakedata_generator.generate_data(sample_size)\n",
+ " if i == 0:\n",
+ " tempresult = cal_metrics.calculate_metrics(y, X, ['HL-H', 'HL-C'],return_numpy=False, df = 10)\n",
+ " keys = list(tempresult.keys())\n",
+ " results.append(np.array(list(tempresult.values())))\n",
+ " else:\n",
+ " tempresult = cal_metrics.calculate_metrics(y, X, ['HL-H', 'HL-C'],return_numpy=True, df = 10)\n",
+ " results.append(tempresult)\n",
+ "results = np.array(results)\n",
+ "\n",
+ "hl_h_pvalue = results[:,1]\n",
+ "hl_c_pvalue = results[:,3]\n",
+ "size_h = np.mean(hl_h_pvalue < 0.05)\n",
+ "size_c = np.mean(hl_c_pvalue < 0.05)\n",
+ "print(\"The size of HL-H with df=M is :\", round(size_h,3))\n",
+ "print(\"The size of HL-C with df=M is :\", round(size_c,3))"
]
},
{
diff --git a/docs/build/html/notebooks/quickstart.html b/docs/build/html/notebooks/quickstart.html
index f3adffa..d4c0377 100644
--- a/docs/build/html/notebooks/quickstart.html
+++ b/docs/build/html/notebooks/quickstart.html
@@ -191,7 +191,8 @@ Command line interface