-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfivestep.html
986 lines (943 loc) · 95.2 KB
/
fivestep.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang="">
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<title>2 Proposal for a Five-Step Method to Elicit Expert Judgment | Dissertation_Duco_Veen.utf8.md</title>
<meta name="description" content="" />
<meta name="generator" content="bookdown 0.14 and GitBook 2.6.7" />
<meta property="og:title" content="2 Proposal for a Five-Step Method to Elicit Expert Judgment | Dissertation_Duco_Veen.utf8.md" />
<meta property="og:type" content="book" />
<meta property="og:url" content="https://github.com/VeenDuco/Dissertation/" />
<meta name="github-repo" content="VeenDuco/Dissertation" />
<meta name="twitter:card" content="summary" />
<meta name="twitter:title" content="2 Proposal for a Five-Step Method to Elicit Expert Judgment | Dissertation_Duco_Veen.utf8.md" />
<meta name="author" content="Duco Veen" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black" />
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon" />
<link rel="prev" href="index.html"/>
<link rel="next" href="DAC1.html"/>
<script src="libs/jquery-2.2.3/jquery.min.js"></script>
<link href="libs/gitbook-2.6.7/css/style.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-table.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-bookdown.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-highlight.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-search.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-fontsettings.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-clipboard.css" rel="stylesheet" />
<style type="text/css">
a.sourceLine { display: inline-block; line-height: 1.25; }
a.sourceLine { pointer-events: none; color: inherit; text-decoration: inherit; }
a.sourceLine:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode { white-space: pre; position: relative; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
code.sourceCode { white-space: pre-wrap; }
a.sourceLine { text-indent: -1em; padding-left: 1em; }
}
pre.numberSource a.sourceLine
{ position: relative; left: -4em; }
pre.numberSource a.sourceLine::before
{ content: attr(title);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; pointer-events: all; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
color: #aaaaaa;
}
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
a.sourceLine::before { text-decoration: underline; }
}
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #7d9029; } /* Attribute */
code span.bn { color: #40a070; } /* BaseN */
code span.bu { } /* BuiltIn */
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4070a0; } /* Char */
code span.cn { color: #880000; } /* Constant */
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
code span.dt { color: #902000; } /* DataType */
code span.dv { color: #40a070; } /* DecVal */
code span.er { color: #ff0000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #40a070; } /* Float */
code span.fu { color: #06287e; } /* Function */
code span.im { } /* Import */
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
code span.op { color: #666666; } /* Operator */
code span.ot { color: #007020; } /* Other */
code span.pp { color: #bc7a00; } /* Preprocessor */
code span.sc { color: #4070a0; } /* SpecialChar */
code span.ss { color: #bb6688; } /* SpecialString */
code span.st { color: #4070a0; } /* String */
code span.va { color: #19177c; } /* Variable */
code span.vs { color: #4070a0; } /* VerbatimString */
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
</style>
</head>
<body>
<div class="book without-animation with-summary font-size-2 font-family-1" data-basepath=".">
<div class="book-summary">
<nav role="navigation">
<ul class="summary">
<li class="chapter" data-level="1" data-path="index.html"><a href="index.html"><i class="fa fa-check"></i><b>1</b> Introduction</a><ul>
<li class="chapter" data-level="1.1" data-path="index.html"><a href="index.html#bayesian-statistics"><i class="fa fa-check"></i><b>1.1</b> Bayesian Statistics</a></li>
<li class="chapter" data-level="1.2" data-path="index.html"><a href="index.html#prior-information"><i class="fa fa-check"></i><b>1.2</b> Prior Information</a></li>
<li class="chapter" data-level="1.3" data-path="index.html"><a href="index.html#expert-elicitation"><i class="fa fa-check"></i><b>1.3</b> Expert Elicitation</a></li>
<li class="chapter" data-level="1.4" data-path="index.html"><a href="index.html#aims-and-outline"><i class="fa fa-check"></i><b>1.4</b> Aims and Outline</a></li>
</ul></li>
<li class="chapter" data-level="2" data-path="fivestep.html"><a href="fivestep.html"><i class="fa fa-check"></i><b>2</b> Proposal for a Five-Step Method to Elicit Expert Judgment</a><ul>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#abstract"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="2.1" data-path="fivestep.html"><a href="fivestep.html#ch02introduction"><i class="fa fa-check"></i><b>2.1</b> Introduction</a></li>
<li class="chapter" data-level="2.2" data-path="fivestep.html"><a href="fivestep.html#five-step-method"><i class="fa fa-check"></i><b>2.2</b> Five-Step Method</a><ul>
<li class="chapter" data-level="2.2.1" data-path="fivestep.html"><a href="fivestep.html#step-1"><i class="fa fa-check"></i><b>2.2.1</b> Step 1</a></li>
<li class="chapter" data-level="2.2.2" data-path="fivestep.html"><a href="fivestep.html#step-2"><i class="fa fa-check"></i><b>2.2.2</b> Step 2</a></li>
<li class="chapter" data-level="2.2.3" data-path="fivestep.html"><a href="fivestep.html#step-3"><i class="fa fa-check"></i><b>2.2.3</b> Step 3</a></li>
<li class="chapter" data-level="2.2.4" data-path="fivestep.html"><a href="fivestep.html#step-4"><i class="fa fa-check"></i><b>2.2.4</b> Step 4</a></li>
<li class="chapter" data-level="2.2.5" data-path="fivestep.html"><a href="fivestep.html#step-5"><i class="fa fa-check"></i><b>2.2.5</b> Step 5</a></li>
</ul></li>
<li class="chapter" data-level="2.3" data-path="fivestep.html"><a href="fivestep.html#elicitation-studies"><i class="fa fa-check"></i><b>2.3</b> Elicitation Studies</a><ul>
<li class="chapter" data-level="2.3.1" data-path="fivestep.html"><a href="fivestep.html#user-feasibility-study"><i class="fa fa-check"></i><b>2.3.1</b> User Feasibility Study</a></li>
<li class="chapter" data-level="2.3.2" data-path="fivestep.html"><a href="fivestep.html#elicitation-staffing-company"><i class="fa fa-check"></i><b>2.3.2</b> Elicitation Staffing Company</a></li>
<li class="chapter" data-level="2.3.3" data-path="fivestep.html"><a href="fivestep.html#elicitation-large-financial-institution"><i class="fa fa-check"></i><b>2.3.3</b> Elicitation Large Financial Institution</a></li>
</ul></li>
<li class="chapter" data-level="2.4" data-path="fivestep.html"><a href="fivestep.html#ch02discussion"><i class="fa fa-check"></i><b>2.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02ethics"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02funding"><i class="fa fa-check"></i>Funding</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02acknowledgments"><i class="fa fa-check"></i>Acknowledgments</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02conflict"><i class="fa fa-check"></i>Conflict of Interest Statement</a></li>
</ul></li>
<li class="chapter" data-level="3" data-path="DAC1.html"><a href="DAC1.html"><i class="fa fa-check"></i><b>3</b> Using the Data Agreement Criterion to Rank Experts’ Beliefs</a><ul>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#abstract-1"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="3.1" data-path="DAC1.html"><a href="DAC1.html#ch03introduction"><i class="fa fa-check"></i><b>3.1</b> Introduction</a></li>
<li class="chapter" data-level="3.2" data-path="DAC1.html"><a href="DAC1.html#expert-data-disagreement"><i class="fa fa-check"></i><b>3.2</b> Expert-Data (Dis)Agreement</a><ul>
<li class="chapter" data-level="3.2.1" data-path="DAC1.html"><a href="DAC1.html#data-agreement-criterion"><i class="fa fa-check"></i><b>3.2.1</b> Data Agreement Criterion</a></li>
<li class="chapter" data-level="3.2.2" data-path="DAC1.html"><a href="DAC1.html#DACvsBF"><i class="fa fa-check"></i><b>3.2.2</b> Comparison to Ranking by the Bayes Factor</a></li>
<li class="chapter" data-level="3.2.3" data-path="DAC1.html"><a href="DAC1.html#DACvsBF2"><i class="fa fa-check"></i><b>3.2.3</b> DAC Versus BF</a></li>
</ul></li>
<li class="chapter" data-level="3.3" data-path="DAC1.html"><a href="DAC1.html#empirical-example"><i class="fa fa-check"></i><b>3.3</b> Empirical Example</a><ul>
<li class="chapter" data-level="3.3.1" data-path="DAC1.html"><a href="DAC1.html#elicitation-procedure"><i class="fa fa-check"></i><b>3.3.1</b> Elicitation Procedure</a></li>
<li class="chapter" data-level="3.3.2" data-path="DAC1.html"><a href="DAC1.html#ranking-the-experts"><i class="fa fa-check"></i><b>3.3.2</b> Ranking the Experts</a></li>
</ul></li>
<li class="chapter" data-level="3.4" data-path="DAC1.html"><a href="DAC1.html#ch03discussion"><i class="fa fa-check"></i><b>3.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03ethics"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03funding"><i class="fa fa-check"></i>Funding</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03acknowledgments"><i class="fa fa-check"></i>Acknowledgments</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03conflict"><i class="fa fa-check"></i>Conflicts of Interest Statement</a></li>
</ul></li>
<li class="chapter" data-level="4" data-path="Hierarchical.html"><a href="Hierarchical.html"><i class="fa fa-check"></i><b>4</b> A Step Forward: Bayesian Hierarchical Modelling as a Tool in Assessment of Individual Discrimination Performance</a><ul>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#abstract-2"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="4.1" data-path="Hierarchical.html"><a href="Hierarchical.html#ch04introduction"><i class="fa fa-check"></i><b>4.1</b> Introduction</a></li>
<li class="chapter" data-level="4.2" data-path="Hierarchical.html"><a href="Hierarchical.html#method"><i class="fa fa-check"></i><b>4.2</b> Method</a><ul>
<li class="chapter" data-level="4.2.1" data-path="Hierarchical.html"><a href="Hierarchical.html#participants"><i class="fa fa-check"></i><b>4.2.1</b> Participants</a></li>
<li class="chapter" data-level="4.2.2" data-path="Hierarchical.html"><a href="Hierarchical.html#stimuli"><i class="fa fa-check"></i><b>4.2.2</b> Stimuli</a></li>
<li class="chapter" data-level="4.2.3" data-path="Hierarchical.html"><a href="Hierarchical.html#procedure"><i class="fa fa-check"></i><b>4.2.3</b> Procedure</a></li>
</ul></li>
<li class="chapter" data-level="4.3" data-path="Hierarchical.html"><a href="Hierarchical.html#results-3"><i class="fa fa-check"></i><b>4.3</b> Results</a><ul>
<li class="chapter" data-level="4.3.1" data-path="Hierarchical.html"><a href="Hierarchical.html#summary-of-the-group-data-published-in-de_klerk_lost_2019"><i class="fa fa-check"></i><b>4.3.1</b> Summary of the group data published in <span class="citation">de Klerk et al. (<span>2019</span>)</span></a></li>
<li class="chapter" data-level="4.3.2" data-path="Hierarchical.html"><a href="Hierarchical.html#data-screening"><i class="fa fa-check"></i><b>4.3.2</b> Data Screening</a></li>
<li class="chapter" data-level="4.3.3" data-path="Hierarchical.html"><a href="Hierarchical.html#analysis-1-linear-regression-model-with-autoregressive-ar1-error-structure"><i class="fa fa-check"></i><b>4.3.3</b> Analysis 1: Linear Regression Model with Autoregressive (AR1) Error Structure</a></li>
<li class="chapter" data-level="4.3.4" data-path="Hierarchical.html"><a href="Hierarchical.html#analysis-2-hierarchical-bayesian-analysis"><i class="fa fa-check"></i><b>4.3.4</b> Analysis 2: Hierarchical Bayesian Analysis</a></li>
</ul></li>
<li class="chapter" data-level="4.4" data-path="Hierarchical.html"><a href="Hierarchical.html#discussion"><i class="fa fa-check"></i><b>4.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch04ethics"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch04acknowledgments"><i class="fa fa-check"></i>Acknowledgments</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch05appendix"><i class="fa fa-check"></i>Appendix A</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch05appendixB"><i class="fa fa-check"></i>Appendix B</a><ul>
<li class="chapter" data-level="4.4.1" data-path="Hierarchical.html"><a href="Hierarchical.html#software"><i class="fa fa-check"></i><b>4.4.1</b> Software</a></li>
<li class="chapter" data-level="4.4.2" data-path="Hierarchical.html"><a href="Hierarchical.html#priors"><i class="fa fa-check"></i><b>4.4.2</b> Priors</a></li>
<li class="chapter" data-level="4.4.3" data-path="Hierarchical.html"><a href="Hierarchical.html#estimation-and-convergence"><i class="fa fa-check"></i><b>4.4.3</b> Estimation and Convergence</a></li>
<li class="chapter" data-level="4.4.4" data-path="Hierarchical.html"><a href="Hierarchical.html#posterior-predictive-check"><i class="fa fa-check"></i><b>4.4.4</b> Posterior predictive check</a></li>
<li class="chapter" data-level="4.4.5" data-path="Hierarchical.html"><a href="Hierarchical.html#sensitivity-analysis"><i class="fa fa-check"></i><b>4.4.5</b> Sensitivity Analysis</a></li>
</ul></li>
</ul></li>
<li class="chapter" data-level="5" data-path="Burns.html"><a href="Burns.html"><i class="fa fa-check"></i><b>5</b> The importance of collaboration in Bayesian analyses with small samples</a><ul>
<li class="chapter" data-level="" data-path="Burns.html"><a href="Burns.html#abstract-3"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="5.1" data-path="Burns.html"><a href="Burns.html#ch05introduction"><i class="fa fa-check"></i><b>5.1</b> Introduction</a></li>
<li class="chapter" data-level="5.2" data-path="Burns.html"><a href="Burns.html#latent-growth-models-with-small-sample-sizes"><i class="fa fa-check"></i><b>5.2</b> Latent Growth Models with small sample sizes</a></li>
<li class="chapter" data-level="5.3" data-path="Burns.html"><a href="Burns.html#empirical-example-analysis-plan"><i class="fa fa-check"></i><b>5.3</b> Empirical example: Analysis plan</a><ul>
<li class="chapter" data-level="5.3.1" data-path="Burns.html"><a href="Burns.html#research-question-model-specification-and-an-overview-of-data"><i class="fa fa-check"></i><b>5.3.1</b> Research question, model specification and an overview of data</a></li>
<li class="chapter" data-level="5.3.2" data-path="Burns.html"><a href="Burns.html#specifying-and-understanding-priors"><i class="fa fa-check"></i><b>5.3.2</b> Specifying and understanding priors</a></li>
</ul></li>
<li class="chapter" data-level="5.4" data-path="Burns.html"><a href="Burns.html#empirical-example-conducting-the-analysis"><i class="fa fa-check"></i><b>5.4</b> Empirical example: Conducting the analysis</a></li>
<li class="chapter" data-level="5.5" data-path="Burns.html"><a href="Burns.html#debugging"><i class="fa fa-check"></i><b>5.5</b> Debugging</a></li>
<li class="chapter" data-level="5.6" data-path="Burns.html"><a href="Burns.html#moving-forward-alternative-models"><i class="fa fa-check"></i><b>5.6</b> Moving forward: Alternative Models</a></li>
<li class="chapter" data-level="5.7" data-path="Burns.html"><a href="Burns.html#conclusion"><i class="fa fa-check"></i><b>5.7</b> Conclusion</a></li>
<li class="chapter" data-level="5.8" data-path="Burns.html"><a href="Burns.html#acknowledgements"><i class="fa fa-check"></i><b>5.8</b> Acknowledgements</a></li>
</ul></li>
<li class="chapter" data-level="6" data-path="elicitlgm.html"><a href="elicitlgm.html"><i class="fa fa-check"></i><b>6</b> Expert Elicitation in the Social Sciences: The case of Posttraumatic Stress Symptoms Development in Children with Burn Injuries</a><ul>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#abstract-4"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="6.1" data-path="elicitlgm.html"><a href="elicitlgm.html#ch06introduction"><i class="fa fa-check"></i><b>6.1</b> Introduction</a></li>
<li class="chapter" data-level="6.2" data-path="elicitlgm.html"><a href="elicitlgm.html#methods"><i class="fa fa-check"></i><b>6.2</b> Methods</a><ul>
<li class="chapter" data-level="6.2.1" data-path="elicitlgm.html"><a href="elicitlgm.html#motivating-example"><i class="fa fa-check"></i><b>6.2.1</b> Motivating Example</a></li>
<li class="chapter" data-level="6.2.2" data-path="elicitlgm.html"><a href="elicitlgm.html#expert-elicitation-1"><i class="fa fa-check"></i><b>6.2.2</b> Expert Elicitation</a></li>
<li class="chapter" data-level="6.2.3" data-path="elicitlgm.html"><a href="elicitlgm.html#sample-of-experts"><i class="fa fa-check"></i><b>6.2.3</b> Sample of Experts</a></li>
</ul></li>
<li class="chapter" data-level="6.3" data-path="elicitlgm.html"><a href="elicitlgm.html#results-4"><i class="fa fa-check"></i><b>6.3</b> Results</a><ul>
<li class="chapter" data-level="6.3.1" data-path="elicitlgm.html"><a href="elicitlgm.html#individual-and-group-expert-judgements"><i class="fa fa-check"></i><b>6.3.1</b> Individual and Group Expert Judgements</a></li>
<li class="chapter" data-level="6.3.2" data-path="elicitlgm.html"><a href="elicitlgm.html#prior-data-disagreement"><i class="fa fa-check"></i><b>6.3.2</b> Prior-Data (dis)Agreement</a></li>
<li class="chapter" data-level="6.3.3" data-path="elicitlgm.html"><a href="elicitlgm.html#audio-recordings"><i class="fa fa-check"></i><b>6.3.3</b> Audio Recordings</a></li>
</ul></li>
<li class="chapter" data-level="6.4" data-path="elicitlgm.html"><a href="elicitlgm.html#discussion-1"><i class="fa fa-check"></i><b>6.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#conflicts-of-interest"><i class="fa fa-check"></i>Conflicts of Interest</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#ethics-statement"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#acknowledgements-1"><i class="fa fa-check"></i>Acknowledgements</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#funding"><i class="fa fa-check"></i>Funding</a></li>
</ul></li>
<li class="chapter" data-level="7" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html"><i class="fa fa-check"></i><b>7</b> Discussion</a><ul>
<li class="chapter" data-level="7.1" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html#hidden-assumptions"><i class="fa fa-check"></i><b>7.1</b> Hidden assumptions</a></li>
<li class="chapter" data-level="7.2" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html#expert-knowledge"><i class="fa fa-check"></i><b>7.2</b> Expert Knowledge</a></li>
<li class="chapter" data-level="7.3" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html#taking-a-decision"><i class="fa fa-check"></i><b>7.3</b> Taking a decision</a></li>
</ul></li>
<li class="chapter" data-level="" data-path="nederlandse-samenvatting.html"><a href="nederlandse-samenvatting.html"><i class="fa fa-check"></i>Nederlandse Samenvatting</a></li>
<li class="chapter" data-level="" data-path="dankwoord.html"><a href="dankwoord.html"><i class="fa fa-check"></i>Dankwoord</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html"><i class="fa fa-check"></i>Curriculum Vitae</a><ul>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#academic-publications"><i class="fa fa-check"></i>Academic Publications</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#book-chapters"><i class="fa fa-check"></i>Book Chapters</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#technical-reports"><i class="fa fa-check"></i>Technical Reports</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#manuscripts-under-review"><i class="fa fa-check"></i>Manuscripts under review</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#grants"><i class="fa fa-check"></i>Grants</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#awards"><i class="fa fa-check"></i>Awards</a></li>
</ul></li>
<li class="chapter" data-level="" data-path="ref.html"><a href="ref.html"><i class="fa fa-check"></i>References</a></li>
</ul>
</nav>
</div>
<div class="book-body">
<div class="body-inner">
<div class="book-header" role="navigation">
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i><a href="./"></a>
</h1>
</div>
<div class="page-wrapper" tabindex="-1" role="main">
<div class="page-inner">
<section class="normal" id="section-">
<div id="fivestep" class="section level1">
<h1><span class="header-section-number">2</span> Proposal for a Five-Step Method to Elicit Expert Judgment</h1>
<div id="abstract" class="section level2 unnumbered">
<h2>Abstract</h2>
<p>Elicitation is a commonly used tool to extract viable information from experts. The information that is held by the expert is extracted and a probabilistic representation of this knowledge is constructed. A promising avenue in psychological research is to incorporated experts’ prior knowledge in the statistical analysis. Systematic reviews on elicitation literature however suggest that it might be inappropriate to directly obtain distributional representations from experts. The literature qualifies experts’ performance on estimating elements of a distribution as unsatisfactory, thus reliably specifying the essential elements of the parameters of interest in one elicitation step seems implausible. Providing feedback within the elicitation process can enhance the quality of the elicitation and interactive software can be used to facilitate the feedback. Therefore, we propose to decompose the elicitation procedure into smaller steps with adjustable outcomes. We represent the tacit knowledge of experts as a location parameter and their uncertainty concerning this knowledge by a scale and shape parameter. Using a feedback procedure, experts can accept the representation of their beliefs or adjust their input. We propose a Five-Step Method which consists of (1) Eliciting the location parameter using the trial roulette method. (2) Provide feedback on the location parameter and ask for confirmation or adjustment. (3) Elicit the scale and shape parameter. (4) Provide feedback on the scale and shape parameter and ask for confirmation or adjustment. (5) Use the elicited and calibrated probability distribution in a statistical analysis and update it with data or to compute a prior-data conflict within a Bayesian framework. User feasibility and internal validity for the Five-Step Method are investigated using three elicitation studies.</p>
<!-- \indent _keywords:_ Bayesian statistics, elicitation, expert judgment, expert knowledge, Five-Step Method, prior -->
</div>
<div id="ch02introduction" class="section level2">
<h2><span class="header-section-number">2.1</span> Introduction</h2>
<blockquote>
<p>“The knowledge held by expert practitioners is too valuable to be ignored. But only when thorough methods are applied, can the application of expert knowledge be as valid as the use of empirical data. The responsibility for the effective and rigorous use of expert knowledge lies with the researchers.”</p>
<p><span class="citation">Drescher et al. (<a href="#ref-drescher_toward_2013" role="doc-biblioref">2013</a>, p. 1)</span></p>
</blockquote>
<p>According to <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> elicitation is the process of extracting and creating a representation of an expert’s beliefs. It can be used for a variety of reasons, e.g., to add information to small sample data <span class="citation">(Kadane, <a href="#ref-kadane_application_1994" role="doc-biblioref">1994</a>; van de Schoot, Sijbrandij, et al., <a href="#ref-van_de_schoot_bayesian_2018" role="doc-biblioref">2018</a>; Zondervan-Zwijnenburg et al., <a href="#ref-zondervan-zwijnenburg_where_2017" role="doc-biblioref">2017</a><a href="#ref-zondervan-zwijnenburg_where_2017" role="doc-biblioref">a</a>)</span>, when there is no data for certain confounding parameters in a model <span class="citation">(Fischer, Lewandowski, & Janssen, <a href="#ref-fischer_estimating_2013" role="doc-biblioref">2013</a>)</span>, when no data is available <span class="citation">(Ho & Smith, <a href="#ref-ho_volcanic_1997" role="doc-biblioref">1997</a>)</span>, as sensitivity analysis to check assumptions about missing data <span class="citation">(Mason et al., <a href="#ref-mason_development_2017" role="doc-biblioref">2017</a>)</span>, or simply to enrich the available data <span class="citation">(Wisniowski, Bijak, & Shang, <a href="#ref-wisniowski_forecasting_2014" role="doc-biblioref">2014</a>)</span>. Expert knowledge is a valuable source of information, as becomes evident in the quote of <span class="citation">Drescher et al. (<a href="#ref-drescher_toward_2013" role="doc-biblioref">2013</a>)</span>. <span class="citation">(Hadorn, Kvizhinadze, Collinson, & Blakely, <a href="#ref-hadorn_useof_2014" role="doc-biblioref">2014</a>)</span> found that 57% of health economic decision models included at least one expert knowledge elicitation parameter, showing that in some fields it is even the norm to use expert elicitation. More examples of elicitation practices in many different fields can be found in overview studies by <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>, Chapter 10)</span> and <span class="citation">Bistline (<a href="#ref-bistline_energy_2014" role="doc-biblioref">2014</a>)</span> or the paper by <span class="citation">Cooke & Goossens (<a href="#ref-cooke_tu_2008" role="doc-biblioref">2008</a>)</span> in which they describe the data base of over 67,000 experts’ subjective probability distributions.</p>
<p>There are many elicitation procedures available, overviews can be found in for instance <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span>, <span class="citation">S. R. Johnson et al. (<a href="#ref-johnson_methods_2010" role="doc-biblioref">2010</a>)</span>, and <span class="citation">Aspinall & Cooke (<a href="#ref-aspinall_quantifying_2013" role="doc-biblioref">2013</a>)</span>. A popular elicitation method is the trial roulette method <span class="citation">(Gore, <a href="#ref-gore_biostatistics_1987" role="doc-biblioref">1987</a>)</span>, sometimes also called the chips and bins method or the histogram method, in which experts assign “chips”" to “bins” of a histogram to ascribe probability. In the procedure, used by for instance <span class="citation">Diamond et al. (<a href="#ref-diamond_expert_2014" role="doc-biblioref">2014</a>)</span> and <span class="citation">Goldstein & Rothschild (<a href="#ref-goldstein_lay_2014" role="doc-biblioref">2014</a>)</span>, the parameter space for which experts can assign probability is divided into equal sections or “bins”. The experts receive 20 “chips”, which are to be distributed amongst these “bins”. For each “chip” that is allocated to one of the “bins”, 5% of the mass of a probability distribution is ascribed. Based on the input provided by the expert, a probability distribution is fitted. The trial roulette method has been validated by <span class="citation">S. R. Johnson, Tomlinson, et al. (<a href="#ref-johnson_valid_2010" role="doc-biblioref">2010</a>)</span> and <span class="citation">M. Zondervan-Zwijnenburg et al. (<a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">2017</a><a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">b</a>)</span> in a face-to-face setting.</p>
<p>Software that can be used in the elicitation with the trial roulette method is available in the MATCH Uncertainty Elicitation Tool <span class="citation">(Morris, Oakley, & Crowe, <a href="#ref-morris_web-based_2014" role="doc-biblioref">2014</a>)</span>. MATCH is an online framework for elicitation procedures. It uses the R-package <span class="citation">(R Core Team, <a href="#ref-r_core_team_r:_2017" role="doc-biblioref">2017</a><a href="#ref-r_core_team_r:_2017" role="doc-biblioref">b</a>)</span> SHELF <span class="citation">(Oakley, <a href="#ref-R-SHELF" role="doc-biblioref">2019</a>)</span> to fit appropriate parametric distributions based on input that is provided by experts.</p>
<p>One of the reasons the trial roulette method is popular is that the procedure provides immediate visual feedback to experts. Feedback is important in elicitation procedures to reduce bias and improve the quality of the elicitation <span class="citation">(Johnson et al., <a href="#ref-johnson_methods_2010" role="doc-biblioref">2010</a>; O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span>. The “chips”" that are allocated in the trial roulette method by the expert visually approximate a probability distribution. However, the feedback provided to the expert is not on the statistical distribution that is actually used by the researcher in the final analyses. It is important to receive conformation of the expert that the interpretation by the researcher matches their beliefs, or as <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>, p. 174)</span> state, <em>“feedback to the expert is the most natural way of evaluating the distribution – the expert is in the best position to judge whether something corresponds to her opinion.”</em> Providing instant feedback on the representation of the experts’ beliefs, based on the input they provide, and how their beliefs are translated into a statistical distribution can easily be done by using software.</p>
<p>Feedback is believed to improve the quality of the elicitation procedure by making experts; reflect and maintain selfconsistency <span class="citation">(Fisher, O’Leary, Low-Choy, Mengersen, & Caley, <a href="#ref-fisher_software_2012" role="doc-biblioref">2012</a>)</span>, by highlighting inconsistencies in judgment and making errors apparent <span class="citation">(Morris et al., <a href="#ref-morris_web-based_2014" role="doc-biblioref">2014</a>; O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> and by allowing for self-correction by experts <span class="citation">(Johnson et al., <a href="#ref-johnson_methods_2010" role="doc-biblioref">2010</a>)</span>. Despite assumed quality improvement by feedback, systematic reviews on elicitation literature by <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> and <span class="citation">Johnson et al. (<a href="#ref-johnson_methods_2010" role="doc-biblioref">2010</a>)</span> conclude that measurement properties of elicitation methods have not been adequately evaluated. Moreover, there is no direct research into how accurate experts can assess properties like the mean, mode, or variance for the distribution of an uncertain parameter. Research by <span class="citation">S. R. Johnson, Tomlinson, et al. (<a href="#ref-johnson_valid_2010" role="doc-biblioref">2010</a>)</span> and <span class="citation">(Zondervan-Zwijnenburg et al., <a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">2017</a><a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">b</a>)</span> provide promising results concerning the trial roulette method. Yet, directly obtaining distributional representations may be inappropriate given experts’ unsatisfactory performance on specifying elements of this distribution. <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> refer to research by <span class="citation">Hofstatter (<a href="#ref-hofstatter_uber_1939" role="doc-biblioref">1939</a>)</span>, <span class="citation">Lathrop (<a href="#ref-lathrop_perceived_1967" role="doc-biblioref">1967</a>)</span> and, <span class="citation">Beach & Scopp (<a href="#ref-beach_intuitive_1968" role="doc-biblioref">1968</a>)</span> to show that experts are not good at interpreting and assigning numerical values to variances and relative variability. It might then be unreasonable to assume that experts are able to reliably specifying a probability distribution in one step.</p>
<p>Therefore, to assist experts in the process of creating a representation of their beliefs in a statistical distribution we propose to decompose the elicitation task in smaller steps to encourage and assist in structured reasoning. Decomposing a problem into more tractable and familiar components is suggested by for instance <span class="citation">Fischhoff (<a href="#ref-fischhoff_debiasing_1982" role="doc-biblioref">1982</a>)</span> to decrease the mismatch between the judge and the task. By decomposing the elicitation task we aim to reduce bias and incorporate more feedback to ensure that experts’ opinions are properly calibrated and represented by the probability distributions that results from the elicitation. In the current paper, the statistical distribution of interest is the skewed normal (SN) distribution<a href="#fn1" class="footnote-ref" id="fnref1"><sup>1</sup></a> because uncertainty might typically not best be captured by a symmetric distribution. This (un)certainty is the key feature of Bayesian statistics, uncertainty reveals the extent of our knowledge and ignorance <span class="citation">(de Finetti, <a href="#ref-de_finetti_theory_1974" role="doc-biblioref">1974</a>)</span>.</p>
<p>We propose the Five-Step Method which consists out of the following steps:</p>
<ol style="list-style-type: decimal">
<li>Elicit the location parameter of the SN using the trial roulette method.</li>
<li>Use software to provide instant feedback on the interpretation of the expert’s beliefs by the researcher so the expert can accept this representation or adjust their input.</li>
<li>Elicit the (un)certainty of the expert by determining the scale and shape parameters of the SN using expert’s statements on the lower and upper bounds for a plausible range of the parameter values.</li>
<li>Use software to provide instant feedback on the interpretation of the expert’s (un)certainty about the location parameter by the researcher so expert can accept this representation or adjust their input.</li>
<li>Use the elicited and calibrated probability distribution in a Bayesian analysis to update it with data or to compute a prior-data conflict.</li>
</ol>
<p>The remainder of the paper is ordered as follows. We first provide details on the Five-Step Method. Thereafter we present a user feasibility study in which we elicited beliefs regarding a trivial sports related question from respondents to investigate visual and procedural preferences of users for the digitized version of the trial roulette method. A second study was carried out by asking experts working at a staffing company about certain key performance indicators which we used to validate the internal validity of steps 1 and 2 of the elicitation procedure. A final study was done with regional directors working at a large financial institution. They provided actual forecasts concerning average turnover per professional in the first quarter of the year 2016 with the Five-Step Method. The participating companies already make predictions concerning the parameters we elicit, yet they do this in the form of point estimates. The experts are thus already used to thinking about these data and predicting these data which makes them highly suitable to include as experts in an elicitation exercise. Yet, it is an extension for them to actively specify and separate knowledge and uncertainty. Because the companies also provided us with data on the predicted parameters we were able to compare the forecasts of the experts with data and thereby get an indication of the internal validity of the elicitation procedure. The proposition to split the elicitation process results in a procedure differing from the existing elicitation procedures as, for example, proposed by <span class="citation">Oakley (<a href="#ref-oakley_eliciting_2010" role="doc-biblioref">2010</a>)</span>, or that can be carried out through the use of existing software like MATCH. Therefore, we programmed our own software. All related materials for this study, including code and data, can be found on the Open Science Framework (OSF) webpage at <a href="https://osf.io/wvujz">https://osf.io/wvujz</a>.</p>
</div>
<div id="five-step-method" class="section level2">
<h2><span class="header-section-number">2.2</span> Five-Step Method</h2>
<p>In this section we describe the technical details of the Five-Step Method which has been programmed in R <span class="citation">(R Core Team, <a href="#ref-r_core_team_r:_2017" role="doc-biblioref">2017</a><a href="#ref-r_core_team_r:_2017" role="doc-biblioref">b</a>)</span> using the shiny package <span class="citation">(Chang, Cheng, Allaire, Xie, & McPherson, <a href="#ref-R-shiny" role="doc-biblioref">2019</a>)</span>.</p>
<div id="step-1" class="section level3">
<h3><span class="header-section-number">2.2.1</span> Step 1</h3>
<p>The first step of the Five-Step Method consists of a digitized version of the trial roulette, which can be seen in Figure <a href="fivestep.html#fig:ch02fig1">2.1</a>. Instead of vertical “bins” a grid is used and the digital “chips” can be placed on the grid. Experts provide estimates for the expected minimum and maximum value of the parameter of interest, represented by the left and rightmost digital “chips” in the grid, based on which the range of the grid is determined. Thereafter they place additional “chips” in the grid. In specific, the input grid, denoted by <strong>G</strong>, is a matrix size 600 (columns) x 300 (rows) and cells are activated by the placement of a digital “chips” in the grid. The cells where a sticker is placed obtain a value of one, all other cells are set to non-available. A second matrix, denoted by <strong>R</strong> of the same dimensions is created in which all rows are equal and the columns are a sequence of numbers with equal intervals running from the reasonable lower to upper bound provided as input. We then create output matrix <strong>O</strong> which contains values from <strong>R</strong> activated by the placement of dots in <strong>G</strong> and after the deletion of all nonavailable values in <strong>O</strong>, the remaining values are stored in a vector.</p>
<div class="figure" style="text-align: center"><span id="fig:ch02fig1"></span>
<img src="figures/chapter_2/Figure_1.PNG" alt="Shiny application for steps 1 and 2. On the left the input fields can be found for the reasonable lower and upper bound as minimum and maximum values. The input grid in which 'chips' can be placed is found on the lower right with the leftmost dot being the minimum value and the right most dot being the maximum value. Further 'chips' are placed by clicking the mouse drawing a maximum of 11 pixels left and right. On the top right feedback is provided, presenting the fitted distribution based on the input." width="85%" />
<p class="caption">
Figure 2.1: Shiny application for steps 1 and 2. On the left the input fields can be found for the reasonable lower and upper bound as minimum and maximum values. The input grid in which ‘chips’ can be placed is found on the lower right with the leftmost dot being the minimum value and the right most dot being the maximum value. Further ‘chips’ are placed by clicking the mouse drawing a maximum of 11 pixels left and right. On the top right feedback is provided, presenting the fitted distribution based on the input.
</p>
</div>
</div>
<div id="step-2" class="section level3">
<h3><span class="header-section-number">2.2.2</span> Step 2</h3>
<p>The vector of values that is elicited in step 1 are used to fit a SN distribution. The SN distribution is defined in this paper as a normal distribution with the additional shape parameter <span class="math inline">\(\gamma\)</span>. The shape parameter is based upon a general method for the transformation of symmetric distributions into skewed distributions as described in <span class="citation">Fernández & Steel (<a href="#ref-fernandez_bayesian_1998" role="doc-biblioref">1998</a>)</span>. The transformation of the symmetric distribution into a skewed distribution is done by allocating mass of the distribution to either side of the mode (<em>M</em>) by controlling the error term (<span class="math inline">\(\epsilon\)</span>) via the following function, taken from Fernandez and Steel Eq. 1:</p>
<p><span class="math display" id="eq:ch02eq1">\[\begin{equation}
p(\epsilon|\gamma) = \frac{2}{{\gamma + \frac{1}{\gamma}}} {f(\frac{\epsilon}{\gamma})I_{(M,\infty)}(\epsilon) + f(\gamma \epsilon)I_{(-\infty,M)}(\epsilon)}.
\tag{2.1}
\end{equation}\]</span></p>
<p>The effect of the shape parameter on the allocation of mass can be seen in Figure <a href="fivestep.html#fig:ch02fig2">2.2</a>. Note that the distributions would be exactly mirrored with respect to the mode if the <span class="math inline">\(\gamma\)</span> values would be <span class="math inline">\(\frac{1}{\gamma}\)</span>.</p>
<p>To fit the SN distribution we make use of the snormFit function from the fGarch package <span class="citation">(Wuertz et al., <a href="#ref-R-fGarch" role="doc-biblioref">2019</a>)</span>. This function uses an optimization algorithm to determine the optimal skewness parameter based on log-likelihood values. The mean and standard deviation are determined based on the vector of elicited values. The mean and standard deviation remain constant and thus there is only one parameter to optimize over, the shape parameter <span class="math inline">\(\gamma\)</span>.</p>
<p>The SN distribution that is fitted based upon the expert’s input is provided as visual feedback to the expert, see Figure <a href="fivestep.html#fig:ch02fig1">2.1</a>. The visual feedback indicates how we interpret the information that is provided by the expert. The expert can accept the representation of their beliefs or adjust input until the representation matches their beliefs. Once the expert approves the representation of their beliefs, the mean value is extracted from the distribution which is to be used in step 3.</p>
<div class="figure" style="text-align: center"><span id="fig:ch02fig2"></span>
<img src="figures/chapter_2/Figure_2.png" alt="Example of the influence of shape parameter $\gamma$ on the allocation of mass for a normal distribution with a variance of 1." width="85%" />
<p class="caption">
Figure 2.2: Example of the influence of shape parameter <span class="math inline">\(\gamma\)</span> on the allocation of mass for a normal distribution with a variance of 1.
</p>
</div>
</div>
<div id="step-3" class="section level3">
<h3><span class="header-section-number">2.2.3</span> Step 3</h3>
<p>Step 3 of the Five-Step Method is used to derive the distributional representation of the expert’s prior beliefs concerning the parameter of interest and can be seen in Figure <a href="fivestep.html#fig:ch02fig3">2.3</a>. We restricted the priors that represent the experts’ beliefs to be SN distributions so <span class="math inline">\(\pi_d \sim SN(\mu_0, \sigma^2_0, \gamma_0)\)</span>, where subscript <em>d</em> denotes expert <span class="math inline">\(d = 1,...,D\)</span>, <span class="math inline">\(\mu_0\)</span> denotes the prior mean, <span class="math inline">\(\sigma^2_0\)</span> denotes the prior variance, and <span class="math inline">\(\gamma_0\)</span> denotes the prior skewness. The value for <span class="math inline">\(\mu_0\)</span> is assumed to be known, either obtained through steps 1 and 2 or stated directly. In step 3 the expert is required to provide values for the reasonable lower and upper bounds they perceive as likely for their estimate of <span class="math inline">\(\mu_0\)</span>. The value for <span class="math inline">\(\mu_0\)</span> is repeated 100 times, the values for the reasonable lower and upper bounds for the estimate are both repeated 10 times.</p>
<div class="figure" style="text-align: center"><span id="fig:ch02fig3"></span>
<img src="figures/chapter_2/Figure_3.PNG" alt="Shiny application for steps 3 and 4. On the left the input fields require entering the estimate for $\mu_0$ and the reasonable lower and upper bound for the estimate. On the right the distribution that is fitted based on the input can be found." width="90%" />
<p class="caption">
Figure 2.3: Shiny application for steps 3 and 4. On the left the input fields require entering the estimate for <span class="math inline">\(\mu_0\)</span> and the reasonable lower and upper bound for the estimate. On the right the distribution that is fitted based on the input can be found.
</p>
</div>
</div>
<div id="step-4" class="section level3">
<h3><span class="header-section-number">2.2.4</span> Step 4</h3>
<p>Based on the input provided in step 3 we will obtain estimates for the scale parameter <span class="math inline">\(\sigma^2_0\)</span> and the shape parameter <span class="math inline">\(\gamma_0\)</span>. The 120 values, <span class="math inline">\(\mu_0\)</span> repeated 100 times and the values for the reasonable lower and upper bounds both repeated 10 times, are provided to the snormFit function by means of which a SN distribution is fitted. The estimates for <span class="math inline">\(\sigma^2_0\)</span> and <span class="math inline">\(\gamma_0\)</span> are obtained and <span class="math inline">\(\mu_0\)</span> is constrained to the input value. Visual feedback is provided to the expert of the resulting SN distribution, which can be seen in Figure <a href="fivestep.html#fig:ch02fig3">2.3</a>. The expert can accept the representation of their beliefs or adjust input until the representation matches their beliefs.</p>
</div>
<div id="step-5" class="section level3">
<h3><span class="header-section-number">2.2.5</span> Step 5</h3>
<p>Use the elicited distribution that represents the expert’s beliefs.</p>
</div>
</div>
<div id="elicitation-studies" class="section level2">
<h2><span class="header-section-number">2.3</span> Elicitation Studies</h2>
<p>In this section we describe the three studies we conducted. During the user feasibility study R version 3.1.2 was used and R version 3.2.3 was used during the elicitations done with the staffing company and the large financial institution. We conducted the elicitations in a semi-structured face-toface setting so that the researcher could provide interpretations accompanying the visual feedback. An advantage of a face-to-face setting is that it allows clarification of procedural and elicitation related questions thereby improving the validity of the responses <span class="citation">(O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span>.</p>
<p><span class="citation">(Cooke & Goossens, <a href="#ref-cooke_procedures_1999" role="doc-biblioref">1999</a>)</span> describe that a panel of four experts can be sufficient for an elicitation, but they recommend a panel of about eight experts as a rule of thumb. In the user feasibility study nine respondents participated. In the staffing company only four experts were available in the entire company, therefore the sample was limited to a size of four. Regarding the study at the large financial institution four experts participated in the end.</p>
<div id="user-feasibility-study" class="section level3">
<h3><span class="header-section-number">2.3.1</span> User Feasibility Study</h3>
<div id="design" class="section level4">
<h4><span class="header-section-number">2.3.1.1</span> Design</h4>
<p>With the user feasibility study we evaluated the usability of the first two steps of the Five-Step Method. Procedural and visual preferences were investigated. Four variations of the shiny application were tested. The respondents (<span class="math inline">\(D=9\)</span>), obtained through convenience sampling from a population of university trained adults, were randomly allocated to two out of the four possible variations of the software.</p>
<p>In the first procedural option, we used the procedure of the trial roulette where 20 digital “chips”, starting with the expected minimal and maximum value, each representing five percent of a distribution, were to be placed in a grid following the procedure described by <span class="citation">Zondervan-Zwijnenburg et al. (<a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">2017</a><a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">b</a>)</span>. After placing 20 “chips” the respondents could submit their input and they were provided with visual feedback on the distribution that was fitted based on these 20 “chips”. They could accept the representation or adjust their input. The second procedural option required the placement of a minimum of seven “chips”, starting with the expected minimal and maximum value. In this procedural variation the distribution that was fitted based on the input was constantly shown. The distribution changed with each placed “chip” and thus instant feedback was provided on the representation of the input. Respondents could, after placing a minimum of seven “chips”, at each point accept the representation of their beliefs or add or adjust input. Next to these two options, we also varied the size of the digital grid in which the “chips”" were placed: large and small.</p>
<p>The respondents evaluated the two variations they were appointed to with a questionnaire asking if the fitted
distribution was a good reflection of their beliefs and what visual and procedural preferences were. Additional questions were based on the taxonomy of <span class="citation">Bloom, Engelhart, Furst, Hill, & Krathwohl (<a href="#ref-bloom_taxonomy_1956" role="doc-biblioref">1956</a>)</span> to identify weak points of the software and procedures. These questions investigated; the comprehension of the instructions, the ability to apply the tool, the understanding of the representation of the “chips”", the relation between input and fitted distribution, and the relation between belief and fitted distribution. The full questionnaire can be found in
the data archive which is available on the OSF webpage at <a href="https://osf.io/wvujz">https://osf.io/wvujz</a>.</p>
</div>
<div id="results" class="section level4">
<h4><span class="header-section-number">2.3.1.2</span> Results</h4>
<p>All respondents indicated that their beliefs where accurately represented. Five of the seven respondents allocated to both procedural variants preferred the second variation. Four of the six respondents allocated to both visual variants preferred the large grid, one abstained from answering. Three out of the nine respondents indicated for at least one of the variations that they did not understand the meaning of the “chips”. In
the first procedural option the “chips” each represented 5% of the data whilst in the second procedural option the meaning depended on the amount of chips that were placed. They allocated mass for the distribution that was fitted. The meaning of the chips was not completely understood by one person who used the first procedure and by two persons who used the second procedure. All three of them used a small grid variation. The three respondents all indicated that they knew what the distribution representing their opinion meant in the end and agreed that this accurately described their view. Based on the results we decided to continue working with the second procedural variation, requiring the minimal placement of seven “chips without further restriction on the number of”chips", and a large grid.</p>
</div>
</div>
<div id="elicitation-staffing-company" class="section level3">
<h3><span class="header-section-number">2.3.2</span> Elicitation Staffing Company</h3>
<div id="design-1" class="section level4">
<h4><span class="header-section-number">2.3.2.1</span> Design</h4>
<p>The goal of the second study was to test the internal validity of elicitations obtained with the first two steps of the Five-Step Method. We found a staffing company willing to participate with experts (<span class="math inline">\(D=4\)</span>) providing predictions about five sales results concerning the first quarter of 2016: contract hours, hourly cost buying and selling, turnover and hourly sales margin. A staffing company is a link between companies that want to hire staff and staff looking to work at companies. They buy work from individuals and thereafter place them to work at other companies. The amount of hours they place an individual at another company in the quarter are the contract hours. The hourly cost buying is what it will cost them per hour to buy the work from the individuals and the hourly cost selling is the price which they charge the companies where they stall the individuals. The turnover is equal to the contract hours multiplied by the hourly cost selling and the hourly sales margin is equal to the hourly cost selling minus the hourly cost buying.</p>
<p>The experts were asked to predict the distribution of the data. In some sectors staffing companies staff a lot of individuals at low margins and thus generate a large turnover. In different sectors they staff few individuals at high margins thereby obtaining the same profit at lower turnover rates. These are all relevant considerations and the experts should know which is the case for their company. The company provided us with actual budgets they made which were indications of carefully constructed predictions. By comparing the predictions of the experts to the budget we could gain an indication of the internal validity of predictions made with the first two steps of the Five-Step Method. If the elicitation results match the budget this indicates that the procedure is able to represent the underlying construct of carefully constructed predictions.</p>
</div>
<div id="results-1" class="section level4">
<h4><span class="header-section-number">2.3.2.2</span> Results</h4>
<p>The results can be found in Figure <a href="fivestep.html#fig:ch02fig4">2.4</a> in which we plotted the predictions of the four experts against the actual budgets for the first quarter. To conceal the true values, which is businesssensitive information, a linear transformation has been done on all variables. It can be seen, especially for the hourly sales margins and the turnover, that experts provided very similar predictions to the budgets, for more detailed information see Table <a href="fivestep.html#tab:ch02tab1">2.1</a>. The resemblance of the predictions to the budget indicates internal validity for the use of the steps 1 and 2 of the Five-Step Method as the elicited predictions closely match carefully constructed expectations. Based on these results we decided not to further adjust the elicitation procedure.</p>
<div class="figure" style="text-align: center"><span id="fig:ch02fig4"></span>
<img src="figures/chapter_2/Figure_4.jpg" alt="Results for elicitation with the staffing company. Experts' predictions plotted with actual budget for contract hours, hourly cost buying and selling, turnover and hourly sales margin concerning the first quarter of 2016." width="75%" />
<p class="caption">
Figure 2.4: Results for elicitation with the staffing company. Experts’ predictions plotted with actual budget for contract hours, hourly cost buying and selling, turnover and hourly sales margin concerning the first quarter of 2016.
</p>
</div>
<div style="page-break-after: always;"></div>
<table style="width:97%;">
<caption><span id="tab:ch02tab1">Table 2.1: </span> Results for elicitation with the staffing company. Experts’ predictions with actual budget for contract hours, hourly cost buying and selling, turnover and hourly sales margin concerning the first quarter of 2016.</caption>
<colgroup>
<col width="34%" />
<col width="15%" />
<col width="11%" />
<col width="15%" />
<col width="20%" />
</colgroup>
<thead>
<tr class="header">
<th></th>
<th></th>
<th align="center"><span class="math inline">\(\mu\)</span></th>
<th align="center"><span class="math inline">\(\sigma\)</span></th>
<th align="center"><span class="math inline">\(\gamma\)</span></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td><strong>Contract</strong>
<strong>Hours</strong></td>
<td>Expert 1</td>
<td align="center">3.88</td>
<td align="center">0.83</td>
<td align="center">4.01*10-6</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 2</td>
<td align="center">3.56</td>
<td align="center">0.61</td>
<td align="center">4.48*10-8</td>
</tr>
<tr class="odd">
<td></td>
<td>Expert 3</td>
<td align="center">3.85</td>
<td align="center">0.70</td>
<td align="center">0.51</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 4</td>
<td align="center">3.34</td>
<td align="center">0.89</td>
<td align="center">0.74</td>
</tr>
<tr class="odd">
<td></td>
<td>Budget</td>
<td align="center">3.37</td>
<td align="center">0.91</td>
<td align="center">5.52*10-4</td>
</tr>
<tr class="even">
<td><br></td>
<td></td>
<td align="center"></td>
<td align="center"></td>
<td align="center"></td>
</tr>
<tr class="odd">
<td><strong>Hourly Cost</strong>
<strong>Buying</strong></td>
<td>Expert 1</td>
<td align="center">3.21</td>
<td align="center">0.99</td>
<td align="center">1.10</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 2</td>
<td align="center">2.74</td>
<td align="center">0.69</td>
<td align="center">1.57</td>
</tr>
<tr class="odd">
<td></td>
<td>Expert 3</td>
<td align="center">2.91</td>
<td align="center">0.80</td>
<td align="center">1.78</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 4</td>
<td align="center">3.45</td>
<td align="center">0.97</td>
<td align="center">7.20</td>
</tr>
<tr class="odd">
<td></td>
<td>Budget</td>
<td align="center">3.09</td>
<td align="center">1.05</td>
<td align="center">566.00</td>
</tr>
<tr class="even">
<td><br></td>
<td></td>
<td align="center"></td>
<td align="center"></td>
<td align="center"></td>
</tr>
<tr class="odd">
<td><strong>Hourly Cost</strong>
<strong>Selling</strong></td>
<td>Expert 1</td>
<td align="center">3.99</td>
<td align="center">1.14</td>
<td align="center">1.73</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 2</td>
<td align="center">3.86</td>
<td align="center">1.04</td>
<td align="center">2.14</td>
</tr>
<tr class="odd">
<td></td>
<td>Expert 3</td>
<td align="center">3.72</td>
<td align="center">0.81</td>
<td align="center">1.41</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 4</td>
<td align="center">4.59</td>
<td align="center">1.43</td>
<td align="center">12.80</td>
</tr>
<tr class="odd">
<td></td>
<td>Budget</td>
<td align="center">3.87</td>
<td align="center">0.99</td>
<td align="center">1.29</td>
</tr>
<tr class="even">
<td><br></td>
<td></td>
<td align="center"></td>
<td align="center"></td>
<td align="center"></td>
</tr>
<tr class="odd">
<td><strong>Turnover</strong></td>
<td>Expert 1</td>
<td align="center">3.39</td>
<td align="center">0.98</td>
<td align="center">1.48</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 2</td>
<td align="center">3.21</td>
<td align="center">0.81</td>
<td align="center">1.53</td>
</tr>
<tr class="odd">
<td></td>
<td>Expert 3</td>
<td align="center">2.71</td>
<td align="center">0.72</td>
<td align="center">0.93</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 4</td>
<td align="center">3.16</td>
<td align="center">1.17</td>
<td align="center">0.98</td>
</tr>
<tr class="odd">
<td></td>
<td>Budget</td>
<td align="center">2.71</td>
<td align="center">0.99</td>
<td align="center">0.76</td>
</tr>
<tr class="even">
<td><br></td>
<td></td>
<td align="center"></td>
<td align="center"></td>
<td align="center"></td>
</tr>
<tr class="odd">
<td><strong>Hourly</strong>
<strong>Sales</strong>
<strong>Margin</strong></td>
<td>Expert 1</td>
<td align="center">2.18</td>
<td align="center">0.94</td>
<td align="center">1.68</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 2</td>
<td align="center">2.46</td>
<td align="center">0.97</td>
<td align="center">1.31</td>
</tr>
<tr class="odd">
<td></td>
<td>Expert 3</td>
<td align="center">2.25</td>
<td align="center">0.76</td>
<td align="center">1.69</td>
</tr>
<tr class="even">
<td></td>
<td>Expert 4</td>
<td align="center">2.18</td>
<td align="center">0.84</td>
<td align="center">1.97</td>
</tr>
<tr class="odd">
<td></td>
<td>Budget</td>
<td align="center">2.06</td>
<td align="center">0.96</td>
<td align="center">1.51</td>
</tr>
</tbody>
</table>
<div style="page-break-after: always;"></div>
</div>
</div>
<div id="elicitation-large-financial-institution" class="section level3">
<h3><span class="header-section-number">2.3.3</span> Elicitation Large Financial Institution</h3>
<p>In the third elicitation study the experts (<span class="math inline">\(D=4\)</span>) were regional directors working at a large financial institution. They are considered experts in knowledge concerning market opportunities, market dynamics and estimating the capabilities of the professionals to seize opportunities. Based on these skills we expected that they could predict the average turnover per professional in the entire country in the first quarter of 2016. In this study the experts did not predict the distribution of the data <span class="math inline">\(\textbf{y}\)</span>, but construct a prior for the mean denoted by <span class="math inline">\(\pi_d(\theta)\)</span>. As <span class="math inline">\(\pi_d(\theta) \sim SN(\mu_0, \sigma^2_0, \gamma_0)\)</span> the elicitation results in the representation of each expert’s beliefs expressed in the hyper parameters <span class="math inline">\(\mu_0\)</span>, <span class="math inline">\(\sigma^2_0\)</span> and <span class="math inline">\(\gamma_0\)</span>. We compare the predictions of the experts against actual results, expressed as the posterior distribution of the average turnover per professional, denoted by <span class="math inline">\(\pi(\theta|y)\)</span>. <span class="math inline">\(\pi(\theta|y) \sim SN(\mu_1, \sigma^2_1, \gamma_1)\)</span>, <span class="math inline">\(\mu_1\)</span> denotes the posterior mean, <span class="math inline">\(\sigma^2_1\)</span> denotes the posterior variance and <span class="math inline">\(\gamma1\)</span> the posterior skewness. The prior for <span class="math inline">\(\pi(\theta|y)\)</span> is a <span class="math inline">\(N(0,100)\)</span> prior which is uninformative given the scale of the data.</p>
<div id="design-2" class="section level4">
<h4><span class="header-section-number">2.3.3.1</span> Design</h4>
<p>The team that participated consisted of 11 experts, 10 regional directors and one director. All were eligible to be included in the study. To comply with conditions set by the Ethics Committee, we ensured that experts whom did not wish to participate could do so without it being known that they refused. Therefore we randomly selected seven out of the 11 experts and invited them to participate. Out of the seven selected experts that we approached, three indicated that they did not want to participate in the study and four indicated that they were willing to participate. All four experts that agreed to participate, did participate and completed the elicitation. The participating experts first performed a practice elicitation for their own sales team before moving on to their estimate for the whole country, enabling them to acquaint themselves with the elicitation applications. Offering this practice elicitation could improve the quality of the elicitations <span class="citation">(Johnson et al., <a href="#ref-johnson_methods_2010" role="doc-biblioref">2010</a>)</span>. Only in the case that the director participated the practice run was be possible. The study receive ethical approval from our internal Ethics Committee of the Faculty of Social and Behavioural Sciences of Utrecht University. The letter of approval can be found in the data archive on the OSF website at <a href="https://osf.io/wvujz">https://osf.io/wvujz</a>.</p>
<p>The Five-Step Method was used in this elicitation study and it consists of the following two parts: the first step is designed to support the expert in the use of reasoned and structured thoughts to obtain an estimate for the location parameter <span class="math inline">\(\mu_0\)</span>. In the second step the estimate for <span class="math inline">\(\mu_0\)</span> is used and the expert is asked to provide a reasonable lower and upper bound for their estimate so the prior distribution for the mean turnover per professional can be constructed.</p>
<p>The “chips” placed in the first step were intended to represent individual professionals in the trial run and
clusters of similar professionals in the elicitation concerning the whole country. Visual feedback was provided on the elicited distribution, accompanied by a description of the value for <span class="math inline">\(\mu_0\)</span> by the researcher. The expert could accept the representation of their beliefs or adjust input until the representation matched their beliefs. Results concerning country wide performance where discussed in terms of total turnover for all professionals within the team, therefore the estimate for <span class="math inline">\(\mu_0\)</span> was transformed using the following function</p>
<p><span class="math display" id="eq:ch02eq2">\[\begin{equation}
\theta^* = a\theta + b,
\tag{2.2}
\end{equation}\]</span>
where <span class="math inline">\(\theta\)</span> represents the parameter of interest and <span class="math inline">\(\theta \sim N(\mu, \sigma^2)\)</span> so that <span class="math inline">\(\theta^* \sim N[a\mu + b, (a\sigma)^2]\)</span>.</p>
<p>The use of the mean as location parameter offered additional options to accommodate differences in reasoning of experts, e.g., a sales expert might feel comfortable to provide estimates for the total turnover of a store, represented by <span class="math inline">\(\theta^*\)</span> in Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a>, but not be comfortable providing estimates for the mean turnover per product sold in the store, represented by <span class="math inline">\(\theta\)</span> in Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a>. By knowing the total amount of products that are sold in the store, entering the amount as value for a and 0 for b in Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a>, the prior beliefs regarding the total turnover can be transformed to prior beliefs regarding mean turnover per product and compared to predictions by other experts. The transformation procedure ensures no expert is forced to adhere to a certain scale. To illustrate this flexibility let us imagine that a store sells nine different types of products and in total sells 104 products. In steps 1 and 2 we wish to elicit and verify the location parameter for the mean turnover. Two experts feel comfortable supplying estimates for turnover per product whilst two other experts only feel comfortable supplying estimates for turnover per product type. They can both adhere to the scale they feel comfortable with as we can use a linear transformation to get them onto the same scale for steps 3 and 4. In Table <a href="fivestep.html#tab:ch02tab2">2.2</a> we supply a numerical example to show how location parameters, elicited on a different scale, can be transformed using Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a> to be on the same scale for steps three and four of the elicitation.</p>
<p>In step 3 of the Five-Step Method, we asked the experts to provide a reasonable lower and upper bound for the total turnover of all professionals: Based on the input a distribution was fitted and visual feedback was provided. The researcher supported the visual feedback with a description explaining that more density on places of the axis indicate more perceived likeliness for that value. The expert could accept the representation of their beliefs or adjust the input for the reasonable lower and upper bound until the representation matched their beliefs. The elicited distribution was transformed back to represent the average turnover per professional using Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a>.</p>
<table style="width:99%;">
<caption><span id="tab:ch02tab2">Table 2.2: </span> Illustration of linear transformations using Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a>. Experts 1 and 2 choose to specify turnover per product, resulting in a location parameter on the product scale. Experts 3 and 4 choose to specify turnover per product type, resulting in a location parameter on the product type scale. In steps 3 and 4 we can use either the location parameter on the total turnover scale or the mean turnover scale for experts to provide a reasonable lower and upper bound. All experts’ elicited location parameters can be transformed to both scales.</caption>
<colgroup>
<col width="19%" />
<col width="19%" />
<col width="20%" />
<col width="20%" />
<col width="20%" />
</colgroup>
<thead>
<tr class="header">
<th></th>
<th align="center">Steps 1 and 2
product scale
mean result
<span class="math inline">\((n=104)\)</span></th>
<th align="center">Steps 1 and 2
product type
scale mean
result <span class="math inline">\((n=9)\)</span></th>
<th align="center">Mean turnover
per product
used in steps
3 and 4</th>
<th align="center">Total turnover
used in steps
3 and 4</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>Expert 1</td>
<td align="center">1.8</td>
<td align="center">-</td>
<td align="center">1.80</td>
<td align="center">187.2</td>
</tr>
<tr class="even">
<td>Expert 2</td>
<td align="center">2.1</td>
<td align="center">–</td>
<td align="center">2.10</td>
<td align="center">218.4</td>
</tr>
<tr class="odd">
<td>Expert 3</td>
<td align="center">-</td>
<td align="center">23</td>
<td align="center">1.99</td>
<td align="center">207</td>
</tr>
<tr class="even">
<td>Expert 4</td>
<td align="center">–</td>
<td align="center">24.5</td>
<td align="center">2.12</td>
<td align="center">220.5</td>
</tr>
</tbody>
</table>
</div>
<div id="results-2" class="section level4">
<h4><span class="header-section-number">2.3.3.2</span> Results</h4>
<p>During the elicitation procedures we noticed that not all experts reasoned in the same way. One expert reasoned for his own region in the expected elements, such that each “chip” represented a professional, but concerning the elicitation for the whole country the “chips” represented regional performances not clusters of professionals that are alike. This deviation did not require an adjustment of procedure just a different value for a in Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a> to obtain the estimate for total turnover used in step 3 of the Five-Step Method. Another expert directly reasoned in total turnover when considering country wide performance and directly provided the estimate used in step 3. A third expert started, especially during the test run concerning the expert’s own team, naming the professionals aloud whilst placing the “chips”,using the expected representations for the input. The procedure proved flexible enough so that each expert could use their own careful reasoning within the same framework and end up with comparable output.</p>
<p>All data were analyzed anonymously and were transformed to avoid revealing business-sensitive information. The elicited priors <span class="math inline">\(\pi_d(\theta)\)</span> can be found in Figure <a href="fivestep.html#fig:ch02fig5">2.5</a>, together with the posterior distribution <span class="math inline">\(\pi(\theta|y)\)</span>. The values for the hyper parameters for <span class="math inline">\(\pi_d(\theta)\)</span> and <span class="math inline">\(\pi(\theta|y)\)</span> can be found in Table <a href="fivestep.html#tab:ch02tab3">2.3</a>. We can see, visually in Figure <a href="fivestep.html#fig:ch02fig5">2.5</a> and numerically in Table <a href="fivestep.html#tab:ch02tab3">2.3</a>, that experts one and two provide very similar predictions, however expert 2 is less uncertain about the prediction. In the same manner we can see that expert four made a prediction that closely resembles the actual realization.</p>
<div class="figure" style="text-align: center"><span id="fig:ch02fig5"></span>
<img src="figures/chapter_2/Figure_5a.png" alt="Results Five-Step Method elicitation study with large financial institution. Elicited expert distributions $\pi_d(\theta)$ plotted with results $\pi(\theta|y)$." width="90%" />
<p class="caption">
Figure 2.5: Results Five-Step Method elicitation study with large financial institution. Elicited expert distributions <span class="math inline">\(\pi_d(\theta)\)</span> plotted with results <span class="math inline">\(\pi(\theta|y)\)</span>.
</p>
</div>
<table style="width:99%;">
<caption><span id="tab:ch02tab3">Table 2.3: </span> The values of the hyper parameters of <span class="math inline">\(\pi(\theta|y)\)</span> and <span class="math inline">\(\pi_d(\theta)\)</span> for the study with the large financial institution.</caption>
<colgroup>
<col width="17%" />
<col width="12%" />
<col width="14%" />
<col width="14%" />
<col width="11%" />
<col width="14%" />
<col width="14%" />
</colgroup>
<thead>
<tr class="header">
<th></th>
<th align="center"><span class="math inline">\(\mu_0\)</span></th>
<th align="center"><span class="math inline">\(\sigma_0\)</span></th>
<th align="center"><span class="math inline">\(\gamma_0\)</span></th>
<th align="center"><span class="math inline">\(\mu_1\)</span></th>
<th align="center"><span class="math inline">\(\sigma_1\)</span></th>
<th align="center"><span class="math inline">\(\gamma_1\)</span></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>Preferred
distribution</td>
<td align="center">–</td>
<td align="center">–</td>
<td align="center">–</td>
<td align="center">2.29</td>
<td align="center">0.10</td>
<td align="center">0.99</td>
</tr>
<tr class="even">
<td>Expert 1</td>
<td align="center">2.15</td>
<td align="center">0.09</td>
<td align="center">0.78</td>
<td align="center">–</td>
<td align="center">–</td>
<td align="center">–</td>
</tr>
<tr class="odd">
<td>Expert 2</td>
<td align="center">2.16</td>
<td align="center">0.07</td>
<td align="center">0.82</td>
<td align="center">–</td>
<td align="center">–</td>
<td align="center">–</td>
</tr>
<tr class="even">
<td>Expert 3</td>
<td align="center">1.97</td>
<td align="center">0.11</td>
<td align="center">0.82</td>
<td align="center">–</td>
<td align="center">–</td>
<td align="center">–</td>
</tr>
<tr class="odd">
<td>Expert 4</td>
<td align="center">2.35</td>
<td align="center">0.11</td>
<td align="center">0.94</td>
<td align="center">–</td>
<td align="center">–</td>
<td align="center">–</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div id="ch02discussion" class="section level2">
<h2><span class="header-section-number">2.4</span> Discussion</h2>
<p>The Five-Step Method provides a first step for eliciting experts in a flexible manner such that no expert is forced to reason on a scale they are uncomfortable with, yet ending up with comparable priors for all experts.</p>
<p>In essence the Five-Step Method resembles the structure for eliciting a distribution as is proposed by <span class="citation">Oakley (<a href="#ref-oakley_eliciting_2010" role="doc-biblioref">2010</a>)</span>. Oakley states four steps, (1) the experts makes some probability judgments about the parameter of interest (2) fit a probability distribution to these judgments (3) provide feedback (4) either accept and use the distribution or repeat steps 3 and 4 based on adjusted input. The difference between what Oakley proposes and the Five-Step Method is that we repeat this cycle twice, once for the elicitation of a location parameter and once for the elicitation of the scale and shape parameters. By decomposing the elicitation task we aim to reduce bias and incorporate more feedback.</p>
<p><span class="citation">Johnson et al. (<a href="#ref-johnson_methods_2010" role="doc-biblioref">2010</a>)</span> concluded that measurement properties of the elicitation methods should be evaluated. We first evaluated usability and thereafter the internal validity for the first two steps of the Five-Step Method. Companies put a lot of effort in carefully constructing their budgets. The study with the staffing company provided evidence to show that experts can produce very similar predictions to the budged using steps 1 and 2 of the Five-Step Method. This high resemblance indicates that the elicited predictions closely reflect predictions made with all available information at hand. Further indications for desirable measurement properties are found when we look at the study with the large financial institution. The data in that case are the actual realization of the average turnover per professional that the experts predicted. Using the Five-Step Method especially expert 4 provided predictions that highly overlap with the actual results, see Figure 5. This provides an indication for the internal validity of the method, experts are able to accurately predict future data using the Five-Step Method. To see if this result holds in general or only in our sample, and to compare the results with other elicitation methods, we recommend a larger study in which experts use multiple elicitation ethods including the Five-Step Method to predict future data.</p>
<p>We acknowledge that asking experts for the reasonable lower and upper bound for their estimate in step 3 of the Five-Step Method could perhaps be an oversimplified procedure and other researchers might prefer to replace this step with eliciting quantiles. <span class="citation">Goldstein & Rothschild (<a href="#ref-goldstein_lay_2014" role="doc-biblioref">2014</a>)</span>, however, found that even laypeople’s intuitions about probability distributions can become quite accurate with the help of graphical elicitation techniques. The finding by Goldstein and Rothschild in combination with our own results from the studies with the staffing company and large financial institution support us in the fact that providing graphical feedback along with the interpretation of the elicited distribution can be a key factor in the calibration of the elicitation. Obtaining confirmation from the experts that the way we represent their beliefs is justified is the crucial element in the proposed Five-Step Method. We follow the same reasoning concerning any possible anchoring bias that is introduced by first eliciting the location parameter of the prior distribution of the expert. We count on the graphical feedback along with the interpretation of the elicited distribution to ensure proper calibration of the elicited distribution. We provided some support for the internal validity of the Five-Step Method, yet to verify the external validity, and reaffirm the internal validity of the Five-Step Method a larger validation study needs to be carried out comparing the Five-Step Method with other elicitation methods.</p>
<p>Besides providing graphical feedback it is desirable to stay as close as possible to the reasoning experts use on a daily basis. The method should be adjusted to fit the expert’s reasoning and not the other way around if we do not want to introduce unnecessary bias. As shown in the study with the large financial institution, the Five-Step Method allows for just that. We can help experts order their thoughts, whether they reason in terms of individuals, regions or totals. All these ways of reasoning can be used by simply altering the value for a in Eq. <a href="fivestep.html#eq:ch02eq2">(2.2)</a> and thereafter transforming the values back to be compared on the same scale.</p>
<p>Using graphical feedback and flexible procedures remains a challenging task in an elicitation process. In the seminal work by <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> it is already recommended that user friendly software should be developed for elicitation purposes, yet each elicitation seems to require a special approach. Even so, it is a worthwhile effort to try and standardize procedures and methods as much as possible so we can work toward a situation that enables applied researchers to use elicitation procedures in their work with ease. We use the R programming language to utilize parametric fitting whilst presenting a web-based interface through the use of the shiny package. We thus use the same building blocks as MATCH. We have taken a first step to show that the Five-Step Method can aid experts in ordering and structuring their thoughts through a systematic and flexible method, tailored to each individual expert and we would welcome the adoption of the method by endeavors such as MATCH.</p>
</div>
<div id="ch02ethics" class="section level2 unnumbered">
<h2>Ethics Statement</h2>
<p>This study was carried out in accordance with the recommendations of the internal Ethics Committee of the Faculty of Social and Behavioural Sciences of Utrecht University, with written informed consent from all subjects. All subjects gave written informed consent in accordance with the Declaration of Helsinki. The protocol was approved by the internal Ethics Committee of the Faculty of Social and Behavioural Sciences of Utrecht University.</p>
</div>
<div id="ch02funding" class="section level2 unnumbered">
<h2>Funding</h2>
<p>The project was supported by a grant from the Netherlands Organization for Scientific Research: NWO-VIDI-452-14-006. MZ-Z is part of the Consortium on Individual Development (CID). The CID is funded through the Gravitation program of the Dutch Ministry of Education, Culture, and Science and the Netherlands Organization for Scientific Research: NWO Gravitation 024-001-003.</p>
</div>
<div id="ch02acknowledgments" class="section level2 unnumbered">
<h2>Acknowledgments</h2>
<p>The authors are grateful to all participants of the elicitation studies for their time, energy, and predictions. Also they would like to thank the participating companies for allowing us access to their resources and information thereby enabling us to provide empirical support for the theoretical work.</p>
</div>
<div id="ch02conflict" class="section level2 unnumbered">
<h2>Conflict of Interest Statement</h2>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</div>
</div>
<h3>References</h3>
<div id="refs" class="references">
<div id="ref-aspinall_quantifying_2013">
<p>Aspinall, W. P., & Cooke, R. M. (2013). Quantifying scientific uncertainty from expert judgement elicitation. In <em>Risk and uncertainty assessment for natural hazards</em> (p. 64). Cambridge University Press Cambridge, UK.</p>
</div>
<div id="ref-beach_intuitive_1968">
<p>Beach, L. R., & Scopp, T. S. (1968). Intuitive statistical inferences about variances. <em>Organ. Behav. Hum. Perform</em>, <em>3</em>, 109–123. doi:<a href="https://doi.org/10.1016/0030-5073(68)90001-9">10.1016/0030-5073(68)90001-9</a></p>
</div>
<div id="ref-bistline_energy_2014">
<p>Bistline, J. E. (2014). Energy technology expert elicitations: An application to natural gas turbine efficiencies. <em>Technological Forecasting and Social Change</em>, <em>86</em>, 177–187.</p>
</div>
<div id="ref-bloom_taxonomy_1956">
<p>Bloom, B. S., Engelhart, M. D., Furst, E. J., Hill, W. H., & Krathwohl, D. R. (1956). <em>Taxonomy of educational objectives: Handbook 1: Cognitive domain</em>. New York, NY: David McKay Co Inc.</p>
</div>
<div id="ref-R-shiny">
<p>Chang, W., Cheng, J., Allaire, J., Xie, Y., & McPherson, J. (2019). <em>Shiny: Web application framework for r</em>. Retrieved from <a href="https://CRAN.R-project.org/package=shiny">https://CRAN.R-project.org/package=shiny</a></p>
</div>
<div id="ref-cooke_tu_2008">
<p>Cooke, R. M., & Goossens, L. H. J. (2008). TU Delft expert judgment data base. <em>Reliability Engineering & System Safety</em>, <em>93</em>(5), 657–674.</p>
</div>
<div id="ref-cooke_procedures_1999">
<p>Cooke, R. M., & Goossens, L. J. H. (1999). <em>Procedures guide for structured expert judgment</em>. Brussels: Commission of the European Communities.</p>
</div>
<div id="ref-de_finetti_theory_1974">
<p>de Finetti, B. (1974). <em>Theory of Probability</em> (Vol. 1 and 2). New York, NY: Wiley.</p>
</div>
<div id="ref-diamond_expert_2014">
<p>Diamond, I. R., Grant, R. C., Feldman, B. M., Tomlinson, G. A., Pencharz, P. B., Ling, S. C., … Wales, P. W. (2014). Expert Beliefs Regarding Novel Lipid-Based Approaches to Pediatric Intestinal Failure-Associated Liver Disease. <em>Journal of Parenteral and Enteral Nutrition</em>, <em>38</em>(6), 702–710.</p>
</div>
<div id="ref-drescher_toward_2013">
<p>Drescher, M., Perera, A. H., Johnson, C. J., Buse, L., Drew, C., & Burgman, M. (2013). Toward rigorous use of expert knowledge in ecological research. <em>Ecosphere</em>, <em>4</em>(7), 1–26.</p>
</div>
<div id="ref-fernandez_bayesian_1998">
<p>Fernández, C., & Steel, M. F. J. (1998). On Bayesian modeling of fat tails and skewness. <em>Journal of the American Statistical Association</em>, <em>93</em>(441), 359–371.</p>
</div>
<div id="ref-fischer_estimating_2013">
<p>Fischer, K., Lewandowski, D., & Janssen, M. (2013). Estimating unknown parameters in haemophilia using expert judgement elicitation. <em>Haemophilia</em>, <em>19</em>(5), e282–e288.</p>
</div>
<div id="ref-fischhoff_debiasing_1982">
<p>Fischhoff, B. (1982). Debiasing. In <em>Judgment under Uncertainty: Heuristics and Biases</em> (pp. 422–444). Cambridge: Cambridge University Press.</p>
</div>
<div id="ref-fisher_software_2012">
<p>Fisher, R., O’Leary, R. A., Low-Choy, S., Mengersen, K., & Caley, M. J. (2012). A software tool for elicitation of expert knowledge about species richness or similar counts. <em>Environmental Modelling & Software</em>, <em>30</em>, 1–14.</p>
</div>
<div id="ref-goldstein_lay_2014">
<p>Goldstein, D. G., & Rothschild, D. (2014). Lay understanding of probability distributions. <em>Judgment & Decision Making</em>, <em>9</em>(1).</p>
</div>
<div id="ref-gore_biostatistics_1987">
<p>Gore, S. (1987). Biostatistics and the medical research council. <em>Med. Res. Council News</em>, <em>35</em>, 19–20.</p>
</div>
<div id="ref-hadorn_useof_2014">
<p>Hadorn, D., Kvizhinadze, G., Collinson, L., & Blakely, T. (2014). Useof expert knowledge elicitation to estimate parameters in health economic decision models. <em>International Journal of Technology Assessment in Health Care</em>, <em>30</em>(4), 461–468.</p>
</div>
<div id="ref-ho_volcanic_1997">
<p>Ho, C.-H., & Smith, E. I. (1997). Volcanic hazard assessment incorporating expert knowledge: Application to the Yucca Mountain region, Nevada, USA. <em>Mathematical Geology</em>, <em>29</em>(5), 615–627.</p>
</div>
<div id="ref-hofstatter_uber_1939">
<p>Hofstatter, P. R. (1939). Uber die schatzung von gruppeneigenschaften. <em>Z. Psychol.</em>, <em>145</em>, 1–44.</p>
</div>
<div id="ref-johnson_methods_2010">
<p>Johnson, S. R., Tomlinson, G. A., Hawker, G. A., Granton, J. T., & Feldman, B. M. (2010). Methods to elicit beliefs for Bayesian priors: A systematic review. <em>Journal of Clinical Epidemiology</em>, <em>63</em>(4), 355–369.</p>
</div>
<div id="ref-johnson_valid_2010">
<p>Johnson, S. R., Tomlinson, G. A., Hawker, G. A., Granton, J. T., Grosbein, H. A., & Feldman, B. M. (2010). A valid and reliable belief elicitation method for Bayesian priors. <em>Journal of Clinical Epidemiology</em>, <em>63</em>(4), 370–383.</p>
</div>
<div id="ref-kadane_application_1994">
<p>Kadane, J. (1994). An application of robust Bayesian analysis to a medical experiment. <em>Journal of Statistical Planning and Inference</em>, <em>40</em>(2-3), 221–232.</p>
</div>
<div id="ref-lathrop_perceived_1967">
<p>Lathrop, R. G. (1967). Perceived variability. <em>Journal of Experimental Psychology</em>, <em>73</em>, 498–502. doi:<a href="https://doi.org/10.1037/h0024344">10.1037/h0024344</a></p>
</div>
<div id="ref-mason_development_2017">
<p>Mason, A. J., Gomes, M., Grieve, R., Ulug, P., Powell, J. T., & Carpenter, J. (2017). Development of a practical approach to expert elicitation for randomised controlled trials with missing health outcomes: Application to the IMPROVE trial. <em>Clinical Trials</em>, <em>14</em>(4), 357–367.</p>
</div>
<div id="ref-morris_web-based_2014">
<p>Morris, D. E., Oakley, J. E., & Crowe, J. A. (2014). A web-based tool for eliciting probability distributions from experts. <em>Environmental Modelling & Software</em>, <em>52</em>, 1–4.</p>
</div>
<div id="ref-oakley_eliciting_2010">
<p>Oakley, J. (2010). Eliciting univariate probability distributions. In <em>Rethinking Risk Measurement and Reporting</em> (Vol. 1). London: Risk Books.</p>
</div>
<div id="ref-R-SHELF">
<p>Oakley, J. (2019). <em>SHELF: Tools to support the sheffield elicitation framework</em>. Retrieved from <a href="https://CRAN.R-project.org/package=SHELF">https://CRAN.R-project.org/package=SHELF</a></p>
</div>
<div id="ref-ohagan_uncertain_2006">
<p>O’Hagan, A., Buck, C. E., Daneshkhah, A., Eiser, J. R., Garthwaite, P. H., Jenkinson, D. J., … Rakow, T. (2006). <em>Uncertain judgements: Eliciting experts’ probabilities</em>. John Wiley & Sons.</p>
</div>
<div id="ref-r_core_team_r:_2017">
<p>R Core Team. (2017b). <em>R: A Language and Environment for Statistical Computing</em>. Vienna, Austria: R Foundation for Statistical Computing. Retrieved from <a href="https://www.R-project.org/">https://www.R-project.org/</a></p>
</div>
<div id="ref-van_de_schoot_bayesian_2018">
<p>van de Schoot, R., Sijbrandij, M., Depaoli, S., Winter, S. D., Olff, M., & van Loey, N. E. (2018). Bayesian PTSD-trajectory analysis with informed priors based on a systematic literature search and expert elicitation. <em>Multivariate Behavioral Research</em>, <em>53</em>(2), 267–291.</p>
</div>
<div id="ref-wisniowski_forecasting_2014">
<p>Wisniowski, A., Bijak, J., & Shang, H. L. (2014). Forecasting Scottish migration in the context of the 2014 constitutional change debate. <em>Popul. Space Place</em>, <em>20</em>, 455–464. doi:<a href="https://doi.org/10.1002/psp.1856">10.1002/psp.1856</a></p>
</div>
<div id="ref-R-fGarch">
<p>Wuertz, D., Setz, T., Chalabi, Y., Boudt, C., Chausse, P., & Miklovac, M. (2019). <em>FGarch: Rmetrics - autoregressive conditional heteroskedastic modelling</em>. Retrieved from <a href="https://CRAN.R-project.org/package=fGarch">https://CRAN.R-project.org/package=fGarch</a></p>
</div>
<div id="ref-zondervan-zwijnenburg_where_2017">
<p>Zondervan-Zwijnenburg, M., Peeters, M., Depaoli, S., & van de Schoot, R. (2017a). Where do priors come from? Applying guidelines to construct informative priors in small sample research. <em>Res. Hum. Dev.</em>, <em>14</em>, 305–320. doi:<a href="https://doi.org/10.1080/15427609.2017.1370966">10.1080/15427609.2017.1370966</a></p>
</div>
<div id="ref-zondervan-zwijnenburg_application_2017">
<p>Zondervan-Zwijnenburg, M., van de Schoot-Hubeek, W., Lek, K., Hoijtink, H., & van de Schoot, R. (2017b). Application and evaluation of an expert judgment elicitation procedure for correlations. <em>Frontiers in Psychology</em>, <em>8</em>, 90.</p>
</div>
</div>
<div class="footnotes">
<hr />
<ol start="1">
<li id="fn1"><p>Using the SN distribution we represent the tacit knowledge of experts by eliciting the location parameter of the distribution, in this case the mean. The uncertainty of the expert about his/her belief on the location parameter is represented by the scale and shape parameter (i.e., variance and skewness of the normal distribution). Eliciting the mean of a normal distribution offers the advantage of easily transformable scale for elicitation procedures. An adjustable scale means that even if one expert reasons in averages and the other expert in sums they can be transformed to be comparable, i.e., let <span class="math inline">\(\theta\)</span> represent the parameter of interest and <span class="math inline">\(\theta \sim N(\mu, \sigma^2)\)</span> and if we transform <span class="math inline">\(\theta\)</span> via the following function <span class="math inline">\(\theta^* = a\theta+b\)</span>, then <span class="math inline">\(\theta^* \sim N[a\mu+b,(a\sigma)^2]\)</span>.<a href="fivestep.html#fnref1" class="footnote-back">↩</a></p></li>
</ol>
</div>
</section>
</div>
</div>
</div>
<a href="index.html" class="navigation navigation-prev " aria-label="Previous page"><i class="fa fa-angle-left"></i></a>
<a href="DAC1.html" class="navigation navigation-next " aria-label="Next page"><i class="fa fa-angle-right"></i></a>
</div>
</div>
<script src="libs/gitbook-2.6.7/js/app.min.js"></script>
<script src="libs/gitbook-2.6.7/js/lunr.js"></script>
<script src="libs/gitbook-2.6.7/js/clipboard.min.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-search.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-sharing.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-fontsettings.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-bookdown.js"></script>
<script src="libs/gitbook-2.6.7/js/jquery.highlight.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-clipboard.js"></script>
<script>
gitbook.require(["gitbook"], function(gitbook) {
gitbook.start({
"sharing": {
"github": false,
"facebook": true,
"twitter": true,
"google": false,
"linkedin": true,
"weibo": false,
"instapaper": false,
"vk": false,
"all": ["facebook", "google", "twitter", "linkedin", "weibo", "instapaper"]
},
"fontsettings": {
"theme": "white",
"family": "sans",
"size": 2
},
"edit": {
"link": null,
"text": null
},
"history": {
"link": null,
"text": null
},
"download": ["Dissertation_Duco_Veen.pdf"],
"toc": {
"collapse": "section"
},
"search": true
});
});
</script>
<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
(function () {
var script = document.createElement("script");
script.type = "text/javascript";
var src = "true";
if (src === "" || src === "true") src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-MML-AM_CHTML";
if (location.protocol !== "file:")
if (/^https?:/.test(src))
src = src.replace(/^https?:/, '');
script.src = src;
document.getElementsByTagName("head")[0].appendChild(script);
})();
</script>
</body>
</html>