-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathelicitlgm.html
752 lines (706 loc) · 95.5 KB
/
elicitlgm.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang="">
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<title>6 Expert Elicitation in the Social Sciences: The case of Posttraumatic Stress Symptoms Development in Children with Burn Injuries | Dissertation_Duco_Veen.utf8.md</title>
<meta name="description" content="" />
<meta name="generator" content="bookdown 0.14 and GitBook 2.6.7" />
<meta property="og:title" content="6 Expert Elicitation in the Social Sciences: The case of Posttraumatic Stress Symptoms Development in Children with Burn Injuries | Dissertation_Duco_Veen.utf8.md" />
<meta property="og:type" content="book" />
<meta property="og:url" content="https://github.com/VeenDuco/Dissertation/" />
<meta name="github-repo" content="VeenDuco/Dissertation" />
<meta name="twitter:card" content="summary" />
<meta name="twitter:title" content="6 Expert Elicitation in the Social Sciences: The case of Posttraumatic Stress Symptoms Development in Children with Burn Injuries | Dissertation_Duco_Veen.utf8.md" />
<meta name="author" content="Duco Veen" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black" />
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon" />
<link rel="prev" href="Burns.html"/>
<link rel="next" href="thesisdiscussion.html"/>
<script src="libs/jquery-2.2.3/jquery.min.js"></script>
<link href="libs/gitbook-2.6.7/css/style.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-table.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-bookdown.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-highlight.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-search.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-fontsettings.css" rel="stylesheet" />
<link href="libs/gitbook-2.6.7/css/plugin-clipboard.css" rel="stylesheet" />
<style type="text/css">
a.sourceLine { display: inline-block; line-height: 1.25; }
a.sourceLine { pointer-events: none; color: inherit; text-decoration: inherit; }
a.sourceLine:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode { white-space: pre; position: relative; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
code.sourceCode { white-space: pre-wrap; }
a.sourceLine { text-indent: -1em; padding-left: 1em; }
}
pre.numberSource a.sourceLine
{ position: relative; left: -4em; }
pre.numberSource a.sourceLine::before
{ content: attr(title);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; pointer-events: all; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
color: #aaaaaa;
}
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
a.sourceLine::before { text-decoration: underline; }
}
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #7d9029; } /* Attribute */
code span.bn { color: #40a070; } /* BaseN */
code span.bu { } /* BuiltIn */
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4070a0; } /* Char */
code span.cn { color: #880000; } /* Constant */
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
code span.dt { color: #902000; } /* DataType */
code span.dv { color: #40a070; } /* DecVal */
code span.er { color: #ff0000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #40a070; } /* Float */
code span.fu { color: #06287e; } /* Function */
code span.im { } /* Import */
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
code span.op { color: #666666; } /* Operator */
code span.ot { color: #007020; } /* Other */
code span.pp { color: #bc7a00; } /* Preprocessor */
code span.sc { color: #4070a0; } /* SpecialChar */
code span.ss { color: #bb6688; } /* SpecialString */
code span.st { color: #4070a0; } /* String */
code span.va { color: #19177c; } /* Variable */
code span.vs { color: #4070a0; } /* VerbatimString */
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
</style>
</head>
<body>
<div class="book without-animation with-summary font-size-2 font-family-1" data-basepath=".">
<div class="book-summary">
<nav role="navigation">
<ul class="summary">
<li class="chapter" data-level="1" data-path="index.html"><a href="index.html"><i class="fa fa-check"></i><b>1</b> Introduction</a><ul>
<li class="chapter" data-level="1.1" data-path="index.html"><a href="index.html#bayesian-statistics"><i class="fa fa-check"></i><b>1.1</b> Bayesian Statistics</a></li>
<li class="chapter" data-level="1.2" data-path="index.html"><a href="index.html#prior-information"><i class="fa fa-check"></i><b>1.2</b> Prior Information</a></li>
<li class="chapter" data-level="1.3" data-path="index.html"><a href="index.html#expert-elicitation"><i class="fa fa-check"></i><b>1.3</b> Expert Elicitation</a></li>
<li class="chapter" data-level="1.4" data-path="index.html"><a href="index.html#aims-and-outline"><i class="fa fa-check"></i><b>1.4</b> Aims and Outline</a></li>
</ul></li>
<li class="chapter" data-level="2" data-path="fivestep.html"><a href="fivestep.html"><i class="fa fa-check"></i><b>2</b> Proposal for a Five-Step Method to Elicit Expert Judgment</a><ul>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#abstract"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="2.1" data-path="fivestep.html"><a href="fivestep.html#ch02introduction"><i class="fa fa-check"></i><b>2.1</b> Introduction</a></li>
<li class="chapter" data-level="2.2" data-path="fivestep.html"><a href="fivestep.html#five-step-method"><i class="fa fa-check"></i><b>2.2</b> Five-Step Method</a><ul>
<li class="chapter" data-level="2.2.1" data-path="fivestep.html"><a href="fivestep.html#step-1"><i class="fa fa-check"></i><b>2.2.1</b> Step 1</a></li>
<li class="chapter" data-level="2.2.2" data-path="fivestep.html"><a href="fivestep.html#step-2"><i class="fa fa-check"></i><b>2.2.2</b> Step 2</a></li>
<li class="chapter" data-level="2.2.3" data-path="fivestep.html"><a href="fivestep.html#step-3"><i class="fa fa-check"></i><b>2.2.3</b> Step 3</a></li>
<li class="chapter" data-level="2.2.4" data-path="fivestep.html"><a href="fivestep.html#step-4"><i class="fa fa-check"></i><b>2.2.4</b> Step 4</a></li>
<li class="chapter" data-level="2.2.5" data-path="fivestep.html"><a href="fivestep.html#step-5"><i class="fa fa-check"></i><b>2.2.5</b> Step 5</a></li>
</ul></li>
<li class="chapter" data-level="2.3" data-path="fivestep.html"><a href="fivestep.html#elicitation-studies"><i class="fa fa-check"></i><b>2.3</b> Elicitation Studies</a><ul>
<li class="chapter" data-level="2.3.1" data-path="fivestep.html"><a href="fivestep.html#user-feasibility-study"><i class="fa fa-check"></i><b>2.3.1</b> User Feasibility Study</a></li>
<li class="chapter" data-level="2.3.2" data-path="fivestep.html"><a href="fivestep.html#elicitation-staffing-company"><i class="fa fa-check"></i><b>2.3.2</b> Elicitation Staffing Company</a></li>
<li class="chapter" data-level="2.3.3" data-path="fivestep.html"><a href="fivestep.html#elicitation-large-financial-institution"><i class="fa fa-check"></i><b>2.3.3</b> Elicitation Large Financial Institution</a></li>
</ul></li>
<li class="chapter" data-level="2.4" data-path="fivestep.html"><a href="fivestep.html#ch02discussion"><i class="fa fa-check"></i><b>2.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02ethics"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02funding"><i class="fa fa-check"></i>Funding</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02acknowledgments"><i class="fa fa-check"></i>Acknowledgments</a></li>
<li class="chapter" data-level="" data-path="fivestep.html"><a href="fivestep.html#ch02conflict"><i class="fa fa-check"></i>Conflict of Interest Statement</a></li>
</ul></li>
<li class="chapter" data-level="3" data-path="DAC1.html"><a href="DAC1.html"><i class="fa fa-check"></i><b>3</b> Using the Data Agreement Criterion to Rank Experts’ Beliefs</a><ul>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#abstract-1"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="3.1" data-path="DAC1.html"><a href="DAC1.html#ch03introduction"><i class="fa fa-check"></i><b>3.1</b> Introduction</a></li>
<li class="chapter" data-level="3.2" data-path="DAC1.html"><a href="DAC1.html#expert-data-disagreement"><i class="fa fa-check"></i><b>3.2</b> Expert-Data (Dis)Agreement</a><ul>
<li class="chapter" data-level="3.2.1" data-path="DAC1.html"><a href="DAC1.html#data-agreement-criterion"><i class="fa fa-check"></i><b>3.2.1</b> Data Agreement Criterion</a></li>
<li class="chapter" data-level="3.2.2" data-path="DAC1.html"><a href="DAC1.html#DACvsBF"><i class="fa fa-check"></i><b>3.2.2</b> Comparison to Ranking by the Bayes Factor</a></li>
<li class="chapter" data-level="3.2.3" data-path="DAC1.html"><a href="DAC1.html#DACvsBF2"><i class="fa fa-check"></i><b>3.2.3</b> DAC Versus BF</a></li>
</ul></li>
<li class="chapter" data-level="3.3" data-path="DAC1.html"><a href="DAC1.html#empirical-example"><i class="fa fa-check"></i><b>3.3</b> Empirical Example</a><ul>
<li class="chapter" data-level="3.3.1" data-path="DAC1.html"><a href="DAC1.html#elicitation-procedure"><i class="fa fa-check"></i><b>3.3.1</b> Elicitation Procedure</a></li>
<li class="chapter" data-level="3.3.2" data-path="DAC1.html"><a href="DAC1.html#ranking-the-experts"><i class="fa fa-check"></i><b>3.3.2</b> Ranking the Experts</a></li>
</ul></li>
<li class="chapter" data-level="3.4" data-path="DAC1.html"><a href="DAC1.html#ch03discussion"><i class="fa fa-check"></i><b>3.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03ethics"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03funding"><i class="fa fa-check"></i>Funding</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03acknowledgments"><i class="fa fa-check"></i>Acknowledgments</a></li>
<li class="chapter" data-level="" data-path="DAC1.html"><a href="DAC1.html#ch03conflict"><i class="fa fa-check"></i>Conflicts of Interest Statement</a></li>
</ul></li>
<li class="chapter" data-level="4" data-path="Hierarchical.html"><a href="Hierarchical.html"><i class="fa fa-check"></i><b>4</b> A Step Forward: Bayesian Hierarchical Modelling as a Tool in Assessment of Individual Discrimination Performance</a><ul>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#abstract-2"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="4.1" data-path="Hierarchical.html"><a href="Hierarchical.html#ch04introduction"><i class="fa fa-check"></i><b>4.1</b> Introduction</a></li>
<li class="chapter" data-level="4.2" data-path="Hierarchical.html"><a href="Hierarchical.html#method"><i class="fa fa-check"></i><b>4.2</b> Method</a><ul>
<li class="chapter" data-level="4.2.1" data-path="Hierarchical.html"><a href="Hierarchical.html#participants"><i class="fa fa-check"></i><b>4.2.1</b> Participants</a></li>
<li class="chapter" data-level="4.2.2" data-path="Hierarchical.html"><a href="Hierarchical.html#stimuli"><i class="fa fa-check"></i><b>4.2.2</b> Stimuli</a></li>
<li class="chapter" data-level="4.2.3" data-path="Hierarchical.html"><a href="Hierarchical.html#procedure"><i class="fa fa-check"></i><b>4.2.3</b> Procedure</a></li>
</ul></li>
<li class="chapter" data-level="4.3" data-path="Hierarchical.html"><a href="Hierarchical.html#results-3"><i class="fa fa-check"></i><b>4.3</b> Results</a><ul>
<li class="chapter" data-level="4.3.1" data-path="Hierarchical.html"><a href="Hierarchical.html#summary-of-the-group-data-published-in-de_klerk_lost_2019"><i class="fa fa-check"></i><b>4.3.1</b> Summary of the group data published in <span class="citation">de Klerk et al. (<span>2019</span>)</span></a></li>
<li class="chapter" data-level="4.3.2" data-path="Hierarchical.html"><a href="Hierarchical.html#data-screening"><i class="fa fa-check"></i><b>4.3.2</b> Data Screening</a></li>
<li class="chapter" data-level="4.3.3" data-path="Hierarchical.html"><a href="Hierarchical.html#analysis-1-linear-regression-model-with-autoregressive-ar1-error-structure"><i class="fa fa-check"></i><b>4.3.3</b> Analysis 1: Linear Regression Model with Autoregressive (AR1) Error Structure</a></li>
<li class="chapter" data-level="4.3.4" data-path="Hierarchical.html"><a href="Hierarchical.html#analysis-2-hierarchical-bayesian-analysis"><i class="fa fa-check"></i><b>4.3.4</b> Analysis 2: Hierarchical Bayesian Analysis</a></li>
</ul></li>
<li class="chapter" data-level="4.4" data-path="Hierarchical.html"><a href="Hierarchical.html#discussion"><i class="fa fa-check"></i><b>4.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch04ethics"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch04acknowledgments"><i class="fa fa-check"></i>Acknowledgments</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch05appendix"><i class="fa fa-check"></i>Appendix A</a></li>
<li class="chapter" data-level="" data-path="Hierarchical.html"><a href="Hierarchical.html#ch05appendixB"><i class="fa fa-check"></i>Appendix B</a><ul>
<li class="chapter" data-level="4.4.1" data-path="Hierarchical.html"><a href="Hierarchical.html#software"><i class="fa fa-check"></i><b>4.4.1</b> Software</a></li>
<li class="chapter" data-level="4.4.2" data-path="Hierarchical.html"><a href="Hierarchical.html#priors"><i class="fa fa-check"></i><b>4.4.2</b> Priors</a></li>
<li class="chapter" data-level="4.4.3" data-path="Hierarchical.html"><a href="Hierarchical.html#estimation-and-convergence"><i class="fa fa-check"></i><b>4.4.3</b> Estimation and Convergence</a></li>
<li class="chapter" data-level="4.4.4" data-path="Hierarchical.html"><a href="Hierarchical.html#posterior-predictive-check"><i class="fa fa-check"></i><b>4.4.4</b> Posterior predictive check</a></li>
<li class="chapter" data-level="4.4.5" data-path="Hierarchical.html"><a href="Hierarchical.html#sensitivity-analysis"><i class="fa fa-check"></i><b>4.4.5</b> Sensitivity Analysis</a></li>
</ul></li>
</ul></li>
<li class="chapter" data-level="5" data-path="Burns.html"><a href="Burns.html"><i class="fa fa-check"></i><b>5</b> The importance of collaboration in Bayesian analyses with small samples</a><ul>
<li class="chapter" data-level="" data-path="Burns.html"><a href="Burns.html#abstract-3"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="5.1" data-path="Burns.html"><a href="Burns.html#ch05introduction"><i class="fa fa-check"></i><b>5.1</b> Introduction</a></li>
<li class="chapter" data-level="5.2" data-path="Burns.html"><a href="Burns.html#latent-growth-models-with-small-sample-sizes"><i class="fa fa-check"></i><b>5.2</b> Latent Growth Models with small sample sizes</a></li>
<li class="chapter" data-level="5.3" data-path="Burns.html"><a href="Burns.html#empirical-example-analysis-plan"><i class="fa fa-check"></i><b>5.3</b> Empirical example: Analysis plan</a><ul>
<li class="chapter" data-level="5.3.1" data-path="Burns.html"><a href="Burns.html#research-question-model-specification-and-an-overview-of-data"><i class="fa fa-check"></i><b>5.3.1</b> Research question, model specification and an overview of data</a></li>
<li class="chapter" data-level="5.3.2" data-path="Burns.html"><a href="Burns.html#specifying-and-understanding-priors"><i class="fa fa-check"></i><b>5.3.2</b> Specifying and understanding priors</a></li>
</ul></li>
<li class="chapter" data-level="5.4" data-path="Burns.html"><a href="Burns.html#empirical-example-conducting-the-analysis"><i class="fa fa-check"></i><b>5.4</b> Empirical example: Conducting the analysis</a></li>
<li class="chapter" data-level="5.5" data-path="Burns.html"><a href="Burns.html#debugging"><i class="fa fa-check"></i><b>5.5</b> Debugging</a></li>
<li class="chapter" data-level="5.6" data-path="Burns.html"><a href="Burns.html#moving-forward-alternative-models"><i class="fa fa-check"></i><b>5.6</b> Moving forward: Alternative Models</a></li>
<li class="chapter" data-level="5.7" data-path="Burns.html"><a href="Burns.html#conclusion"><i class="fa fa-check"></i><b>5.7</b> Conclusion</a></li>
<li class="chapter" data-level="5.8" data-path="Burns.html"><a href="Burns.html#acknowledgements"><i class="fa fa-check"></i><b>5.8</b> Acknowledgements</a></li>
</ul></li>
<li class="chapter" data-level="6" data-path="elicitlgm.html"><a href="elicitlgm.html"><i class="fa fa-check"></i><b>6</b> Expert Elicitation in the Social Sciences: The case of Posttraumatic Stress Symptoms Development in Children with Burn Injuries</a><ul>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#abstract-4"><i class="fa fa-check"></i>Abstract</a></li>
<li class="chapter" data-level="6.1" data-path="elicitlgm.html"><a href="elicitlgm.html#ch06introduction"><i class="fa fa-check"></i><b>6.1</b> Introduction</a></li>
<li class="chapter" data-level="6.2" data-path="elicitlgm.html"><a href="elicitlgm.html#methods"><i class="fa fa-check"></i><b>6.2</b> Methods</a><ul>
<li class="chapter" data-level="6.2.1" data-path="elicitlgm.html"><a href="elicitlgm.html#motivating-example"><i class="fa fa-check"></i><b>6.2.1</b> Motivating Example</a></li>
<li class="chapter" data-level="6.2.2" data-path="elicitlgm.html"><a href="elicitlgm.html#expert-elicitation-1"><i class="fa fa-check"></i><b>6.2.2</b> Expert Elicitation</a></li>
<li class="chapter" data-level="6.2.3" data-path="elicitlgm.html"><a href="elicitlgm.html#sample-of-experts"><i class="fa fa-check"></i><b>6.2.3</b> Sample of Experts</a></li>
</ul></li>
<li class="chapter" data-level="6.3" data-path="elicitlgm.html"><a href="elicitlgm.html#results-4"><i class="fa fa-check"></i><b>6.3</b> Results</a><ul>
<li class="chapter" data-level="6.3.1" data-path="elicitlgm.html"><a href="elicitlgm.html#individual-and-group-expert-judgements"><i class="fa fa-check"></i><b>6.3.1</b> Individual and Group Expert Judgements</a></li>
<li class="chapter" data-level="6.3.2" data-path="elicitlgm.html"><a href="elicitlgm.html#prior-data-disagreement"><i class="fa fa-check"></i><b>6.3.2</b> Prior-Data (dis)Agreement</a></li>
<li class="chapter" data-level="6.3.3" data-path="elicitlgm.html"><a href="elicitlgm.html#audio-recordings"><i class="fa fa-check"></i><b>6.3.3</b> Audio Recordings</a></li>
</ul></li>
<li class="chapter" data-level="6.4" data-path="elicitlgm.html"><a href="elicitlgm.html#discussion-1"><i class="fa fa-check"></i><b>6.4</b> Discussion</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#conflicts-of-interest"><i class="fa fa-check"></i>Conflicts of Interest</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#ethics-statement"><i class="fa fa-check"></i>Ethics Statement</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#acknowledgements-1"><i class="fa fa-check"></i>Acknowledgements</a></li>
<li class="chapter" data-level="" data-path="elicitlgm.html"><a href="elicitlgm.html#funding"><i class="fa fa-check"></i>Funding</a></li>
</ul></li>
<li class="chapter" data-level="7" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html"><i class="fa fa-check"></i><b>7</b> Discussion</a><ul>
<li class="chapter" data-level="7.1" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html#hidden-assumptions"><i class="fa fa-check"></i><b>7.1</b> Hidden assumptions</a></li>
<li class="chapter" data-level="7.2" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html#expert-knowledge"><i class="fa fa-check"></i><b>7.2</b> Expert Knowledge</a></li>
<li class="chapter" data-level="7.3" data-path="thesisdiscussion.html"><a href="thesisdiscussion.html#taking-a-decision"><i class="fa fa-check"></i><b>7.3</b> Taking a decision</a></li>
</ul></li>
<li class="chapter" data-level="" data-path="nederlandse-samenvatting.html"><a href="nederlandse-samenvatting.html"><i class="fa fa-check"></i>Nederlandse Samenvatting</a></li>
<li class="chapter" data-level="" data-path="dankwoord.html"><a href="dankwoord.html"><i class="fa fa-check"></i>Dankwoord</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html"><i class="fa fa-check"></i>Curriculum Vitae</a><ul>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#academic-publications"><i class="fa fa-check"></i>Academic Publications</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#book-chapters"><i class="fa fa-check"></i>Book Chapters</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#technical-reports"><i class="fa fa-check"></i>Technical Reports</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#manuscripts-under-review"><i class="fa fa-check"></i>Manuscripts under review</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#grants"><i class="fa fa-check"></i>Grants</a></li>
<li class="chapter" data-level="" data-path="curriculum-vitae.html"><a href="curriculum-vitae.html#awards"><i class="fa fa-check"></i>Awards</a></li>
</ul></li>
<li class="chapter" data-level="" data-path="ref.html"><a href="ref.html"><i class="fa fa-check"></i>References</a></li>
</ul>
</nav>
</div>
<div class="book-body">
<div class="body-inner">
<div class="book-header" role="navigation">
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i><a href="./"></a>
</h1>
</div>
<div class="page-wrapper" tabindex="-1" role="main">
<div class="page-inner">
<section class="normal" id="section-">
<div id="elicitlgm" class="section level1">
<h1><span class="header-section-number">6</span> Expert Elicitation in the Social Sciences: The case of Posttraumatic Stress Symptoms Development in Children with Burn Injuries</h1>
<div id="abstract-4" class="section level2 unnumbered">
<h2>Abstract</h2>
<p>Experts provide an alternative source of information to classical data collection methods such as surveys. They can provide additional insight into problems, supplement existing data or provide insights when classical data collection is troublesome. In this paper we explore the (dis)similarities between expert judgements and data collected by traditional data collection methods regarding the development of Posttraumatic Stress Symptoms (PTSS) in children with burn injuries. By means of an elicitation procedure the experts’ domain expertise are formalized and represented in the form of probability distributions. The method is used to obtain beliefs from 14 experts, including nurses and psychologists, and those beliefs are contrasted with questionnaire data collected by <span class="citation">Egberts, van de Schoot, Geenen, & van Loey (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> on the same issue. The individual and aggregated expert judgements are contrasted with the questionnaire data by means of Kullback-Leibler divergences. The aggregated judgements of the group that mainly includes psychologists resembles the questionnaire data more than almost all individual experts.</p>
<!-- \indent _keywords:_ Bayesian statistics, elicitation, expert judgment, expert knowledge, Latent Growth Curve Model, prior, prior-data (dis)agreement -->
</div>
<div id="ch06introduction" class="section level2">
<h2><span class="header-section-number">6.1</span> Introduction</h2>
<p>Expert elicitation entails the extraction of information from experts and the translation of this information into a probabilistic representation and there are many reasons to elicit expert knowledge. For instance to supplement existing data using priors that are informed by expert knowledge <span class="citation">(van de Schoot, Sijbrandij, et al., <a href="#ref-van_de_schoot_bayesian_2018" role="doc-biblioref">2018</a>)</span>. Alternatively, some data have information gaps that can be filled by means of expert judgements <span class="citation">(Dodd, Yuen, Sismanidis, Seddon, & Jenkins, <a href="#ref-dodd_global_2017" role="doc-biblioref">2017</a>; Fischer et al., <a href="#ref-fischer_estimating_2013" role="doc-biblioref">2013</a>)</span> or when data are present expert judgements can serve as a quality control for the data <span class="citation">(Veen, Stoel, Schalken, Mulder, & van de Schoot, <a href="#ref-veen_using_2018" role="doc-biblioref">2018</a>)</span>. Elicitation can also be used for forecasting purposes <span class="citation">(Murphy & Winkler, <a href="#ref-murphy_subjective_1974" role="doc-biblioref">1974</a>, <a href="#ref-murphy_probability_1984" role="doc-biblioref">1984</a>)</span> or when there is no data available at all <span class="citation">(Hald et al., <a href="#ref-hald_world_2016" role="doc-biblioref">2016</a>; Ho & Smith, <a href="#ref-ho_volcanic_1997" role="doc-biblioref">1997</a>)</span>. The use of expert knowledge is widespread across many disciplines. To give some examples, <span class="citation">Dodd et al. (<a href="#ref-dodd_global_2017" role="doc-biblioref">2017</a>)</span> elicited expert based estimates for case-fatality ratios in HIV-positive children with tuberculosis who did not receive treatment. <span class="citation">Barons et al. (<a href="#ref-barons_eliciting_2018" role="doc-biblioref">2018</a>)</span> describe the use of expert judgements to create decision support systems with an example in food security and <span class="citation">Dewispelare, Herren, & Clemen (<a href="#ref-dewispelare_use_1995" role="doc-biblioref">1995</a>)</span> describe expert elicitation in relation to the long-term behavior of high-level nuclear waste repositories. For numerous other examples on elicitation practices see for instance Chapter 10 of <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> listing applications in sales, medicine, nuclear industry, veterinary science and many more. Alternatively, examples using a specific elicitation tool are given in <span class="citation">Gosling (<a href="#ref-gosling_shelf:_2018" role="doc-biblioref">2018</a>)</span>, or see <span class="citation">Cooke & Goossens (<a href="#ref-cooke_tu_2008" role="doc-biblioref">2008</a>)</span> who describe a data base of over 67,000 elicited judgements.</p>
<p>Recently, there is a growing interest in the use of expert elicitation in the social sciences. Where <span class="citation">van de Schoot et al. (<a href="#ref-van_de_schoot_systematic_2017" role="doc-biblioref">2017</a>)</span> only found two cases that reported the use of expert opinions to inform priors in 25 years of Bayesian statistics in psychology, this trend might slowly be changing. <span class="citation">Gronau, Ly, & Wagenmakers (<a href="#ref-gronau_informed_2019" role="doc-biblioref">2019</a>)</span> elicited expert judgements on effects sizes such that these could be used in informed Bayesian t-tests, in their example related to a replication study in the field of psychology. <span class="citation">Lek & van de Schoot (<a href="#ref-lek_development_2018" role="doc-biblioref">2018</a>)</span> elicited prior distribution from teachers concerning the math abilities of their students. <span class="citation">Zondervan-Zwijnenburg et al. (<a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">2017</a><a href="#ref-zondervan-zwijnenburg_application_2017" role="doc-biblioref">b</a>)</span> elicited expert judgements on the correlation between cognitive potential and academic performance. Moreover, methods are being developed to facilitate expert elicitation in a flexible manner such that experts are guided in the elicitation process <span class="citation">(Veen et al., <a href="#ref-veen_proposal_2017" role="doc-biblioref">2017</a>)</span>.</p>
<p>Whatever the reasons of the elicitation, the goal is to get an accurate representation of the experts’ beliefs and associated (un)certainty and enable the representation of the experts’ domain knowledge in terms of a probability distribution. Overconfidence of experts is one of the crucial issues in expert elicitation <span class="citation">(O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span>, resulting in elicited probability distributions with little uncertainty. In the seminal work of <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> feedback is named as the most natural way to improve the accuracy of elicited beliefs and interactive software as almost essential for the effective use of feedback. This is corroborated by <span class="citation">Goldstein & Rothschild (<a href="#ref-goldstein_lay_2014" role="doc-biblioref">2014</a>)</span> who found that visual feedback can increase even laypeople’s intuitions about probability distributions. Over a decade has passed since the advice by <span class="citation">O’Hagan et al. (<a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span> and many have followed the advice. Elicitation software can be split into more general and more custom variations. Some more general frameworks are, for instance, ElicitN which was developed by <span class="citation">Fisher et al. (<a href="#ref-fisher_software_2012" role="doc-biblioref">2012</a>)</span> for the elicitation of count data. <span class="citation">Truong, Heuvelink, & Gosling (<a href="#ref-truong_web-based_2013" role="doc-biblioref">2013</a>)</span> made a web-based tool for the elicitation of variogram estimates which describe a degree of spacial dependence. The elicitator was developed for indirect elicitation, creating a scenario based elicitation <span class="citation">(James, Choy, & Mengersen, <a href="#ref-james_elicitator:_2010" role="doc-biblioref">2010</a>; Low-Choy, James, Murray, & Mengersen, <a href="#ref-low-choy_elicitator:_2012" role="doc-biblioref">2012</a>)</span>. <span class="citation">Morris et al. (<a href="#ref-morris_web-based_2014" role="doc-biblioref">2014</a>)</span> developed MATCH which is based on the R package SHELF <span class="citation">(Oakley, <a href="#ref-R-SHELF" role="doc-biblioref">2019</a>)</span> and which is a very general elicitation tool that allows multiple elicitation methods to be used interactively to elicit single parameters. <span class="citation">Garthwaite, Al-Awadhi, Elfadaly, & Jenkinson (<a href="#ref-garthwaite_prior_2013" role="doc-biblioref">2013</a>)</span> developed an elicitation procedure for generalized linear and piecewise-linear models, <span class="citation">Runge, Scherbaum, Curtis, & Riggelsen (<a href="#ref-runge_interactive_2013" role="doc-biblioref">2013</a>)</span> for seismic-hazard analysis and <span class="citation">Elfadaly & Garthwaite (<a href="#ref-elfadaly_eliciting_2017" role="doc-biblioref">2017</a>)</span> for eliciting Dirichlet and Gaussian copula prior distributions. Sometimes more customized software is developed for specific elicitation settings <span class="citation">(e.g. Bojke et al., <a href="#ref-bojke_eliciting_2010" role="doc-biblioref">2010</a>; Haakma, Steuten, Bojke, & IJzerman, <a href="#ref-haakma_belief_2014" role="doc-biblioref">2014</a>; Hampson, Whitehead, Eleftheriou, & Brogan, <a href="#ref-hampson_bayesian_2014" role="doc-biblioref">2014</a>; Hampson et al., <a href="#ref-hampson_elicitation_2015" role="doc-biblioref">2015</a>)</span>. The use of software, customized or not, to increase the accuracy of the elicited beliefs is now common practice.</p>
<p>In this paper we present an elicitation methodology especially designed for eliciting parameters of a Latent Growth Curve Model (LGM) regarding the development of Posttraumatic Stress Symptoms (PTSS) in children with burn injuries. LGMs are commonly used to analyze longitudinal data, especially in the social sciences <span class="citation">(e.g. Buist, Dekovic, Meeus, & van Aken, <a href="#ref-buist_developmental_2002" role="doc-biblioref">2002</a>; Catts, Bridges, Little, & Tomblin, <a href="#ref-catts_reading_2008" role="doc-biblioref">2008</a>; Orth, Robins, & Widaman, <a href="#ref-orth_life-span_2012" role="doc-biblioref">2012</a>)</span>. These models include repeated measurements of observed variables, and allow researchers to examine change or development over time in the construct of interest. For extensive explanations of LGMs see <span class="citation">Duncan & Duncan (<a href="#ref-duncan_introduction_2004" role="doc-biblioref">2004</a>)</span>, <span class="citation">Little (<a href="#ref-little_longitudinal_2013" role="doc-biblioref">2013</a>)</span>, and <span class="citation">Little, Bovaird, & Slegers (<a href="#ref-little_methods_2006" role="doc-biblioref">2006</a>)</span>. Because the incidence of severe burn injuries in school-aged children and adolescents is relatively low, obtaining a sufficient sample to estimate LGMs is challenging. Nevertheless, to gain knowledge on the development of posttraumatic stress symptoms in this group of children, these types of models are favored over simpler models. Expert elicitation might provide and alternative, or supplement, to data collection for cases like our motivating example where traditional data is sparse.</p>
<p>The main aim of this paper is to compare domain expertise, expressed by experts in an elicitation setting, to data on the same topic collected by means of traditional data collection methods. Comparing experts’ domain knowledge to traditional data collection methods can provide unique insights into the topic of interest and the perception thereof. In the remainder of this paper we first describe the methodology that is used to elicit the expert judgements. The methodology is an extension of the Five-Step Method <span class="citation">(Veen et al., <a href="#ref-veen_proposal_2017" role="doc-biblioref">2017</a>)</span>, adapted to elicited multiple parameters. We elicit expert judgements from 14 experts, including nurses and psychologists. Thereafter, we compare individual expert judgements, aggregated group level expert judgements, and data collected by mean of traditional methods with one another and reflect on the elicitation procedure. Finally, we conclude the paper with a discussion section including recommendations for future research. All related materials for this study, including code and data, can be found on the Open Science Framework (OSF) webpage for this project at <a href="https://osf.io/y5evf/">https://osf.io/y5evf/</a>.</p>
</div>
<div id="methods" class="section level2">
<h2><span class="header-section-number">6.2</span> Methods</h2>
<p>In the first section we describe the motivating example for this study. In the next section we elaborate on the elicitation procedure and software that has been developed. Finally, we describe the sample of experts (<span class="math inline">\(N=14\)</span>) participating in the elicitation study. The study receive ethical approval from our internal Ethics Committee of the Faculty of Social and Behavioural Sciences of Utrecht University. The letter of approval can be found in the data archive on the <a href="https://osf.io/y5evf/">OSF website for this project</a>.</p>
<div id="motivating-example" class="section level3">
<h3><span class="header-section-number">6.2.1</span> Motivating Example</h3>
<p>The motivating example for this paper is the development of Posttraumatic Stress Symptoms (PTSS) in children after a burn event. In a prospective study on child and parent adjustment after pediatric burns, data on these symptoms were collected in three Dutch and four Belgian burn centers. Children aged 8-18 years old were eligible to participate in the study if they had been hospitalized for more than 24 hours and if the percentage total body surface area (TBSA) burned was at least 1%. In <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span>, a more detailed description of the overall study and sample can be found here. This sample consists of 100 children that reported on their symptoms of traumatic stress within the first month after the burn event (T1), and subsequently at 3 (T2) months post-burn. For the purpose of the current study, we also included the measures obtained 12 months (T3) post-burn. Children filled out the Children’s Responses to Trauma Inventory (CRTI, revised version; <span class="citation">Alisic, Eland, & Kleber (<a href="#ref-alisic_childrens_2006" role="doc-biblioref">2006</a>)</span>). This measure assesses four symptom clusters of posttraumatic stress, including intrusion (e.g., repetitive, intrusive recollections of the trauma), avoidance (e.g., avoiding conversations of the event), arousal (e.g., difficulty concentrating), and other child-specific responses (e.g., feelings of guilt). Further details on this measure can be found in <span class="citation">Alisic, Eland, Huijbregts, & Kleber (<a href="#ref-alisic_manual_2011" role="doc-biblioref">2011</a>)</span>.</p>
<p>As the current study includes three measurements of PTSS at different time points a straightforward model to analyse the development of PTSS symptoms is an LGM. Figure <a href="elicitlgm.html#fig:ch06fig1">6.1</a> provides a visual representation of an LGM for this motivating example. The model is parameterized such that the latent intercept provides an estimate for PTSS in the first month after the burn event. The latent slope describes the change in PTSS one year post-burn. Parameterizing the slope by year instead of per month is done to ease the reasoning in the elicitation procedure. Furthermore, the scale of the PTSS scores has been standardized for the data of the prospective study and for the elicitation study. The scores can fall between 0-100. A zero score means that none of the symptoms of any of the clusters of posttraumatic stress are present. A score of 100 means that all symptoms from all clusters are present to their maximum extent. A standardized cut-off value of 42 was used to indicate clinical relevance of symptoms and corresponds to the cut-off value provided in the CRTI manual.</p>
<div class="figure" style="text-align: center"><span id="fig:ch06fig1"></span>
<img src="Dissertation_Duco_Veen_files/figure-html/ch06fig1-1.png" alt="Visual representation of a Latent Growth Curve Model with three observed time points for PTSS. " width="50%" />
<p class="caption">
Figure 6.1: Visual representation of a Latent Growth Curve Model with three observed time points for PTSS.
</p>
</div>
</div>
<div id="expert-elicitation-1" class="section level3">
<h3><span class="header-section-number">6.2.2</span> Expert Elicitation</h3>
<p>To optimally prepare the experts within the limited time that was allocated for each elicitation a short introduction was presented by the researchers conducting the elicitation (DV & ME), hereafter named the facilitators. The facilitators presented the experts with a brief overview of what expert elicitation is, what it can be used for and how to interpret the probability distributions that are used to represent their beliefs. Thereafter, to familiarize the experts with the elicitation procedure itself, an example elicitation for an unrelated topic was presented to the experts using the same elicitation tool. After the example elicitation the facilitators introduced the specifics related to the motivating example and the actual elicitation. The population, measurement scale, CRTI with the relevant symptom clusturs, and research question were introduced and experts were invited to ask questions to clarify any part of the procedure. Once the experts stated that they were ready to continue with the elicitation they were requested to sign the informed consent letter, which they received prior to the elicitation. If they agreed, they also agreed to the recording of the elicitation procedure. The experts were requested to reason aloud during the elicitation. The recordings were transcribed to provide additional insights in the elicitation procedure and possible differences between experts. The experts carried out the elicitation procedure using the software that is described next.</p>
<p>The software and procedure in this study was based on the Five-Step Method, developed by <span class="citation">Veen et al. (<a href="#ref-veen_proposal_2017" role="doc-biblioref">2017</a>)</span>, with a slight adaptation to elicit multiple parameters instead of a single parameter. The Five-Step Method decomposes the elicitation process in multiple smaller steps, providing visual feedback at each stage of the elicitation procedure. By decomposing the elicitation task and providing visual feedback the procedures aims to reduce bias, for instance from overconfidence. The software has been developed in the form of a shiny web application <span class="citation">(Chang et al., <a href="#ref-R-shiny" role="doc-biblioref">2019</a>)</span>. Using shiny to develop elicitation tools is not uncommon, see for instance <span class="citation">Hampson et al. (<a href="#ref-hampson_bayesian_2014" role="doc-biblioref">2014</a>)</span>, <span class="citation">Hampson et al. (<a href="#ref-hampson_elicitation_2015" role="doc-biblioref">2015</a>)</span> and the original Five-Step Method by <span class="citation">Veen et al. (<a href="#ref-veen_proposal_2017" role="doc-biblioref">2017</a>)</span>. In what follows we describe the Five-Step Method as implemented for this specific study, note that steps 3 and 4 were repeated for each parameter.</p>
<p><strong>Step 1.</strong> Ten individual PTSS trajectories were elicited for an LGM. These individual trajectories should be representative for the population. From these individual trajectories we could deduce information on the point estimates for the average intercept and average slope parameters. This first step is called indirect elicitation because no statement is required directly concerning the parameters of interest. Figure <a href="elicitlgm.html#fig:ch06fig2">6.2</a> provides a visual representation of step one.</p>
<p><strong>Step 2.</strong> Feedback was provided on the average trajectory that was based upon the ten individual trajectories that the expert provided. The expert could accept this as the average trajectory, and thereby accept point estimate for the average intercept and slope, or the expert could adjust their input in step one. Figure <a href="elicitlgm.html#fig:ch06fig3">6.3</a> provides a visual representation of step two.</p>
<p><strong>Step 3.</strong> The experts provided a reasonable lowerbound and upperbound for the point estimates of the group mean intercept and the group mean slope that were obtained using steps one and two. The lowerbound and upperbound were used to determine the scale and shape of the probability distribution that was used to represent the experts’ beliefs. This is called direct elicitation because the experts provided information directly related to the parameters of interest.</p>
<p><strong>Step 4.</strong> Feedback was provided on the probability distribution that was used to represent the experts’ beliefs. Figure <a href="elicitlgm.html#fig:ch06fig4">6.4</a> provides a visual representation of steps 3 and 4 with respect to the average intercept, top panel, and the average slope, bottom panel. Both single parameter feedback was provided, in the form of a probability distribution, as well as the effect on the implied average trajectory. The experts could accept the representation of their beliefs or adjust their input in step three.</p>
<p><strong>Step 5.</strong> The experts were shown a summary page on the elicitation, see Figure <a href="elicitlgm.html#fig:ch06fig5">6.5</a>. If the experts accepted the representation of their beliefs the probability distributions were now ready to be saved and used in the analyses.</p>
</div>
<div id="sample-of-experts" class="section level3">
<h3><span class="header-section-number">6.2.3</span> Sample of Experts</h3>
<p>Fourteen experts participated in the elicitation study. Experts from all three Dutch burn centers were included. These experts had different professions, including (child) psychologists, pediatric nurses, specialized nurses for burn injuries, and nurses with an additional masters degree (MSc). During the process of obtaining this degree, these nurses worked together closely with psychologist and observed their work. Because reporting the individual expert professions would remove almost all anonymity, we ensured that no elicited probability distributions can be associated with individual experts and therefore categorized the experts into two groups. The first group consisted of experts who have obtained an MSc degree (<span class="math inline">\(N=7\)</span>) and the second group consisted of experts who have not (<span class="math inline">\(N=7\)</span>). As the former group mostly consisted of psychologists or experts who have undergone at least some education in psychology we shall refer to this group as the psychologists. The later group consisted mostly of nurses with a variety of additional specializations and we shall refer to this group as the nurses. The groups we elicited judgements from are considered large enough for elicitation studies. <span class="citation">Cooke & Goossens (<a href="#ref-cooke_procedures_1999" role="doc-biblioref">1999</a>)</span> recommend to use the largest possible number of experts, but at least four. We were able to include seven experts in both groups of experts.</p>
<div style="page-break-after: always;"></div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig2"></span>
<img src="figures/chapter_6/Figure2b.PNG" alt="Step one of the elicitation procedure. Trajectories of PTSS development were elicited for 10 individuals that are representatitve for the population. From these trajectories point estiamtes for the average intercept and the average slope were obtained." width="90%" />
<p class="caption">
Figure 6.2: Step one of the elicitation procedure. Trajectories of PTSS development were elicited for 10 individuals that are representatitve for the population. From these trajectories point estiamtes for the average intercept and the average slope were obtained.
</p>
</div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig3"></span>
<img src="figures/chapter_6/Figure3.PNG" alt="Step two in the elicitation procedure, providing visual feedback on the extracted average trajectory based upon the experts' provided individual trajectories." width="90%" />
<p class="caption">
Figure 6.3: Step two in the elicitation procedure, providing visual feedback on the extracted average trajectory based upon the experts’ provided individual trajectories.
</p>
</div>
<div style="page-break-after: always;"></div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig4"></span>
<img src="figures/chapter_6/Figure4a.PNG" alt="Steps three and four of elicitation procedure for the average intercept, top panel, and the average slope, bottom panel. The input that was required for step three was provided in the fields on the top left of the tab in the elicitation software. The single parameter feedback was provided on the bottom left of the tab, displaying the fitted distribution with respect to that parameter. The effect on the implied average trajectory was displayed on the right hand side of the tab. The average trajectory that was accepted in step two is diplayed and a gray band has been added around this average trajectory that represents the 95\% Credibility Interval (CI) for the average trajectory. In the top panel only the uncerainty with respect to the intercept was added to the average trajectory, in the bottom panel the uncertainty with respect to both the interecept and the slope was added." width="90%" /><img src="figures/chapter_6/Figure4b.PNG" alt="Steps three and four of elicitation procedure for the average intercept, top panel, and the average slope, bottom panel. The input that was required for step three was provided in the fields on the top left of the tab in the elicitation software. The single parameter feedback was provided on the bottom left of the tab, displaying the fitted distribution with respect to that parameter. The effect on the implied average trajectory was displayed on the right hand side of the tab. The average trajectory that was accepted in step two is diplayed and a gray band has been added around this average trajectory that represents the 95\% Credibility Interval (CI) for the average trajectory. In the top panel only the uncerainty with respect to the intercept was added to the average trajectory, in the bottom panel the uncertainty with respect to both the interecept and the slope was added." width="90%" />
<p class="caption">
Figure 6.4: Steps three and four of elicitation procedure for the average intercept, top panel, and the average slope, bottom panel. The input that was required for step three was provided in the fields on the top left of the tab in the elicitation software. The single parameter feedback was provided on the bottom left of the tab, displaying the fitted distribution with respect to that parameter. The effect on the implied average trajectory was displayed on the right hand side of the tab. The average trajectory that was accepted in step two is diplayed and a gray band has been added around this average trajectory that represents the 95% Credibility Interval (CI) for the average trajectory. In the top panel only the uncerainty with respect to the intercept was added to the average trajectory, in the bottom panel the uncertainty with respect to both the interecept and the slope was added.
</p>
</div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig5"></span>
<img src="figures/chapter_6/Figure5.PNG" alt="Summary page of the elicitation procedure. The top left plot within the page displayed all individual trajectories that the expert specified. The top right plot displayed the average trajectory that was obtained based on those individual trajectories. The bottom left plot displayed the average trajectory with uncertainty (95\% CI) concerning the intercept value taken into account. The bottom right plot displayed the average trajectory with uncertainty (95\% CI) concerning both the intercept value and the slope value taken into account." width="90%" />
<p class="caption">
Figure 6.5: Summary page of the elicitation procedure. The top left plot within the page displayed all individual trajectories that the expert specified. The top right plot displayed the average trajectory that was obtained based on those individual trajectories. The bottom left plot displayed the average trajectory with uncertainty (95% CI) concerning the intercept value taken into account. The bottom right plot displayed the average trajectory with uncertainty (95% CI) concerning both the intercept value and the slope value taken into account.
</p>
</div>
</div>
</div>
<div id="results-4" class="section level2">
<h2><span class="header-section-number">6.3</span> Results</h2>
<p>This section first covers a descriptive part on the expert judgements. We report the priors that the experts provided, the mixture priors that can be made from these expert judgements on an aggregated and group distinct level. Thereafter we report prior-data (dis)agreement measures for all individual expert judgements and the mixture distributions. These prior-data (dis)agreements are based upon the data that was collected in the prospective study by <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span>. Finally, we report notable results from the audio recordings. Note that the quantitative results, analyses and an overview of individual expert judgements can be found via the OSF webpage for this project at <a href="https://osf.io/y5evf/">https://osf.io/y5evf/</a>. The transcripts of the audio recordings include many identifying characteristics with respect to both the experts and patients they described during the elicitation and to preserve privacy these are not available. This is in accordance with the ethical approval agreement.</p>
<div id="individual-and-group-expert-judgements" class="section level3">
<h3><span class="header-section-number">6.3.1</span> Individual and Group Expert Judgements</h3>
<p>All 14 expert judgement had been elicited allowing them to specify a skewed normal distribution parameterized according to <span class="citation">Burkner (<a href="#ref-burkner_parameterization_2019" role="doc-biblioref">2019</a>)</span>. In Figure <a href="elicitlgm.html#fig:ch06fig6">6.6</a> all the elicited individual expert distributions can be found as well as the mixture distributions<a href="#fn2" class="footnote-ref" id="fnref2"><sup>2</sup></a> for all experts, the psychologists group and the nurses group regarding both the mean intercept and the mean slope of PTSS development. From Figure <a href="elicitlgm.html#fig:ch06fig6">6.6</a> it can be seen that the experts judgements differed quite substantially. Especially concerning the development of PTSS as expressed by the slope parameter we can see that actually expert disagreed on the direction of the effect and with a lot of confidence. When we look at the groups of experts an interesting pattern emerges. If we combine the expert judgements of the psychologists and the nurses into their respective groups the nurses turn out to have a substantially different view from the psychologists. Not only did the nurses’ judgements express a higher initial amount of PTSS to be present in the population on average, their combined view also expressed that these initial PTSS scores are quite likely to increase on average over time. The psychologists in contrast assigned almost no probability to an increase in the average PTSS score over the time period of a year, see Figure <a href="elicitlgm.html#fig:ch06fig7">6.7</a> for a closer look.</p>
</div>
<div id="prior-data-disagreement" class="section level3">
<h3><span class="header-section-number">6.3.2</span> Prior-Data (dis)Agreement</h3>
<p>To assess the (dis)agreement of experts’ judgements with the data from the prospective study by <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> we used Kullback-Leibler (KL) divergences <span class="citation">(Kullback & Leibler, <a href="#ref-kullback_information_1951" role="doc-biblioref">1951</a>)</span> between the posterior distribution that is based upon the data and an uninformative benchmark prior and the individual and aggregated expert judgements. Using information theoretical distance measures to asses prior-data (dis)agreement in this manner has previously been discussed by for instance <span class="citation">Bousquet (<a href="#ref-bousquet_diagnostics_2008" role="doc-biblioref">2008</a>)</span>, <span class="citation">Lek & van de Schoot (<a href="#ref-lek_how_2019" role="doc-biblioref">2019</a>)</span>, and <span class="citation">Veen et al. (<a href="#ref-veen_using_2018" role="doc-biblioref">2018</a>)</span>. KL-divergences provide us with an indication of how much information is lost is we approximate distribution <span class="math inline">\(\pi_1\)</span> by another distribution <span class="math inline">\(\pi_2\)</span>. A higher divergence indicates a higher loss of information. In this case <span class="math inline">\(\pi_1\)</span> will be the posterior distribution based upon the data and an uninformative benchmark prior, to which we refer as the reference posterior. We approximate the reference posterior with the elicited prior distributions and report the loss of information. For an overview of the priors that are used to compute the reference posterior see Figure <a href="elicitlgm.html#fig:ch06fig8">6.8</a> and in Figure <a href="elicitlgm.html#fig:ch06fig9">6.9</a> the reference posteriors for the group mean latent intercept and slope are visualized.</p>
<p>In addition to comparing the expert priors to the benchmark posterior we added two other comparisons to create a frame of reference. Two benchmark situations are added and their loss of information is calculated. In the situation of benchmark 1 we would take some information regarding the measurement instrument into account. The scale of the measurement instrument was standardized such that values are between 0 and 100, therefore a <span class="math inline">\(U(0, 100)\)</span> prior on the group mean intercept would cover all possible parameter values. With the parameterization such that the final time measurement implies a change of one times the individuals latent slope parameter, taking the standardized scale into account a <span class="math inline">\(U(-100, 100)\)</span> prior on the latent slope covers all possible parameter values and declares them equally possible. For benchmark 2 we take an extremely conservative approach and take two <span class="math inline">\(N(0, 10^8)\)</span> priors on the latent group mean intercept and slope.</p>
<p>The KL-divergences are reported in <a href="elicitlgm.html#tab:ch06tab1">6.1</a> and are the numerical representation of the loss of information that occurs by approximating the reference posteriors distributions from Figure <a href="elicitlgm.html#fig:ch06fig9">6.9</a> by the distributions that can be seen in Figure <a href="elicitlgm.html#fig:ch06fig7">6.7</a> for the experts priors. It seems that there are quite some experts that are in disagreement with the collected data from <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span>. There are some individuals, notably experts 9 and 13, that provide a very similar view to the collected data, whilst some experts provide a similar view with respect to one of the two parameters, e.g. experts 3 and 6. It is notable that the group of psychologists in particular and the group of experts as a whole show less loss of information with respect than the data than most experts on both parameters. Finally, what is noteworthy is that benchmark 1, which has no preference for any part of the parameter space covered by the measurement instrument, resembles the data more than most experts and more than the nurses as a group.</p>
</div>
<div id="audio-recordings" class="section level3">
<h3><span class="header-section-number">6.3.3</span> Audio Recordings</h3>
<p>The following observations were noteworthy in the transcripts of the audio recordings. All psychologists referred back to the concept of PTSS specifically during the elicitation procedure. In the group of nurses we found a lot of mentioning of stress, but only two nurses actually referred back to PTSS specifically. Three psychologists reflected on the linearity assumption of the model and noted that non-linear trajectories often occur. Five of the nurses expressed sentiments that the more severe cases came to mind more easily and therefore might be over represented in their beliefs. Only one psychologists expressed a similar statement. Thee experts, one psychologist and two nurses, actively reflected on the visual feedback and adjusted their input in the elicitation tool based on this. One experts, a nurse, stated that although they were sure about the direction of the trajectory they felt unsure about the associated numerical representation. Finally, one expert, a nurse, repeatedly mentioned that they found the task hard to do.</p>
<div class="figure" style="text-align: center"><span id="fig:ch06fig6"></span>
<img src="figures/chapter_6/all_priors.png" alt="Elicited prior distributions from all experts and the associated mixture priors for all experts, the psychologists group and the nurses group regarding both the mean intercept and the mean slope of PTSS development." width="100%" />
<p class="caption">
Figure 6.6: Elicited prior distributions from all experts and the associated mixture priors for all experts, the psychologists group and the nurses group regarding both the mean intercept and the mean slope of PTSS development.
</p>
</div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig7"></span>
<img src="figures/chapter_6/priors_experts_slope.png" alt="Elicited prior distributions from all experts and the associated mixture priors for all experts, the psychologists group and the nurses group regarding the mean slope of PTSS development. There was a notable difference in expert judgements between the psychologists and the nurses groups." width="85%" />
<p class="caption">
Figure 6.7: Elicited prior distributions from all experts and the associated mixture priors for all experts, the psychologists group and the nurses group regarding the mean slope of PTSS development. There was a notable difference in expert judgements between the psychologists and the nurses groups.
</p>
</div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig8"></span>
<img src="figures/chapter_6/priors_reference.png" alt="Visual representation of the prior distributions that are used to obtain the reference posterior. The prior distributions are \(\alpha_1 \sim N(0, 10^8)\), \(\alpha_2 \sim N(0, 10^8)\), \(\psi_{11} \sim half-t(3, 0, 196)\), \(\psi_{22} \sim half-t(3, 0, 196)\), \(\psi_{21} \sim U(-1, 1)\), and \(\theta \sim half-t(3, 0, 196)\)." width="100%" />
<p class="caption">
Figure 6.8: Visual representation of the prior distributions that are used to obtain the reference posterior. The prior distributions are <span class="math inline">\(\alpha_1 \sim N(0, 10^8)\)</span>, <span class="math inline">\(\alpha_2 \sim N(0, 10^8)\)</span>, <span class="math inline">\(\psi_{11} \sim half-t(3, 0, 196)\)</span>, <span class="math inline">\(\psi_{22} \sim half-t(3, 0, 196)\)</span>, <span class="math inline">\(\psi_{21} \sim U(-1, 1)\)</span>, and <span class="math inline">\(\theta \sim half-t(3, 0, 196)\)</span>.
</p>
</div>
<div class="figure" style="text-align: center"><span id="fig:ch06fig9"></span>
<img src="figures/chapter_6/experts_vs_reference.png" alt="Visual representation of the reference posterior distributions for the group mean of latent intercept and slope with the group expert priors for the parameters. The reference posteriors are approximately distributed \(\alpha_1 \sim N(22.7, 1.3)\), and \(\alpha_2 \sim N(-14.6, 1.9)\)." width="100%" />
<p class="caption">
Figure 6.9: Visual representation of the reference posterior distributions for the group mean of latent intercept and slope with the group expert priors for the parameters. The reference posteriors are approximately distributed <span class="math inline">\(\alpha_1 \sim N(22.7, 1.3)\)</span>, and <span class="math inline">\(\alpha_2 \sim N(-14.6, 1.9)\)</span>.
</p>
</div>
<table>
<caption><span id="tab:ch06tab1">Table 6.1: </span>Kullback-Leibler divergences for all individual and mixture priors to the reference posterior.</caption>
<thead>
<tr class="header">
<th></th>
<th align="right">Intercept</th>
<th align="right">Slope</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>Benchmark 1</td>
<td align="right">3.04</td>
<td align="right">3.56</td>
</tr>
<tr class="even">
<td>Benchmark 2</td>
<td align="right">8.56</td>
<td align="right">8.39</td>
</tr>
<tr class="odd">
<td>Nurses</td>
<td align="right">8.19</td>
<td align="right">5.88</td>
</tr>
<tr class="even">
<td>Psychologists</td>
<td align="right">1.99</td>
<td align="right">2.18</td>
</tr>
<tr class="odd">
<td>All</td>
<td align="right">2.72</td>
<td align="right">2.63</td>
</tr>
<tr class="even">
<td>Expert 1</td>
<td align="right">42.87</td>
<td align="right">59.18</td>
</tr>
<tr class="odd">
<td>Expert 2</td>
<td align="right">45.16</td>
<td align="right">25.87</td>
</tr>
<tr class="even">
<td>Expert 3</td>
<td align="right">6.71</td>
<td align="right">1.23</td>
</tr>
<tr class="odd">
<td>Expert 4</td>
<td align="right">72.86</td>
<td align="right">55.38</td>
</tr>
<tr class="even">
<td>Expert 5</td>
<td align="right">5.66</td>
<td align="right">98.32</td>
</tr>
<tr class="odd">
<td>Expert 6</td>
<td align="right">2.10</td>
<td align="right">22.17</td>
</tr>
<tr class="even">
<td>Expert 7</td>
<td align="right">79.20</td>
<td align="right">59.61</td>
</tr>
<tr class="odd">
<td>Expert 8</td>
<td align="right">46.97</td>
<td align="right">4.37</td>
</tr>
<tr class="even">
<td>Expert 9</td>
<td align="right">2.48</td>
<td align="right">1.28</td>
</tr>
<tr class="odd">
<td>Expert 10</td>
<td align="right">43.74</td>
<td align="right">67.55</td>
</tr>
<tr class="even">
<td>Expert 11</td>
<td align="right">12.78</td>
<td align="right">64.56</td>
</tr>
<tr class="odd">
<td>Expert 12</td>
<td align="right">99.94</td>
<td align="right">4.88</td>
</tr>
<tr class="even">
<td>Expert 13</td>
<td align="right">0.35</td>
<td align="right">3.62</td>
</tr>
<tr class="odd">
<td>Expert 14</td>
<td align="right">75.00</td>
<td align="right">74.11</td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="discussion-1" class="section level2">
<h2><span class="header-section-number">6.4</span> Discussion</h2>
<p>We were able to elicit expert judgements with respect to the development of PTSS in young burn victims from 14 experts and contrasted this with data collected in a traditional way by means of a questionnaire. This study demonstrated differences in views between experts. On an individual basis the experts were particularly in disagreement with regards to the change of PTSS one year post-burn. Many experts do not overlap with each other in their beliefs when we look at the elicited probability distributions for the slope parameter. The expert judgements not only differed from one individual to the next, but it may be that there is a relationship between the experts’ role in the post-burn treatment process and their view on the childrens’ development of PTSS. The two groups of experts notably differed in the aggregated elicited judgements. Moreover, the aggregated judgements of the psychologists seemed to align with the data collected by <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> whilst the nurses judgements seemed to differ.</p>
<p>With respect to the differences between the two groups of experts the most remarkable difference was found with respect to the slope parameter. The aggregated views of the groups of experts result in distributions with more uncertainty compared to the individual experts’ beliefs. The dispersed views of the experts put together ensure coverage of a larger part of the parameters space than the individual expert judgements do. Interestingly, the more uncertain distributions still clearly present a separation of the views regarding the development of PTSS in young burn victims between the nurses expert group on the one hand and the psychologists expert group and the data collected by <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> on the other hand. The aggregated judgements from the psychologists assigned almost no probability to the group average PTSS increasing one year post-burn. The aggregated judgements from the nurses, in contrast, assigned a lot of probability to an increase of the group average PTSS one year post-burn. As there is no grounded truth that we cannot conclude which views are a better or worse representation. However, the results do indicate that the nurses and the psychologists are not in agreement on what happens with respect to the development of PTSS in young burn victims. The audio recordings of the elicitation settings provided a possible explanation for this important distinction. All psychologists at some point during the elicitation referred to, or specifically mentioned, the construct of PTSS. The group of nurses mentioned several sources of distress but only two nurses actually referred back to PTSS, one of which also judged the one year post-burn PTSS to decrease. As burn victims indeed can experience other sources of distress, e.g. related to the development of scar tissue or operations they have to undergo, nurses might have convoluted PTSS symptoms with other patient symptoms. This could also explain why the aggregated nurses’ view judged the initial PTSS level to be higher for the group average than the aggregated psychologists’ view. Overall, the differences might reflect the fact that psychologists are trained to diagnose and treat PTSS, whereas nurses are primarily concerned with procedural and physical care for the patient, and are not responsible for diagnosing and treating PTSS.</p>
<p>Besides differences between the nurses and the psychologists we also found a substantial difference between the reference posteriors, that provided a representation of the data from <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span>, and the aggregated nurses prior. In Figure <a href="elicitlgm.html#fig:ch06fig9">6.9</a> it can be seen that the psychologists’ views overlapped with the reference posteriors. The nurses’ views however did almost not overlap with reference posteriors. This could also numerically be assessed, as was done with the KL-divergences in Table <a href="elicitlgm.html#tab:ch06tab1">6.1</a>. Because the aggregated nurses prior had little overlap with the reference posteriors the Benchmark 1 priors, uniform priors that takes the information of the measurement instrument into account, outperformed this group in terms of loss of information. This implies the data collected by <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> was better approximated by an uninformed expression of the measurement properties of the questionnaire than by the nurses’ group prior. The children in the study by <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> expressed a lower quantity of PTSS in their self-reported questionnaires compared to the nurses’ expert judgements on PTSS symptoms for this populations. There can be several explanations for this discrepancy, first the questionnaire might have resulted in underreporting of symptoms, a view also expressed by one of the experts. In line with this, <span class="citation">Egberts et al. (<a href="#ref-egberts_mother_2018" role="doc-biblioref">2018</a>)</span> found that mothers gave higher ratings of their child’s PTSS compared to the children themselves, although mothers’ ratings also appeared to be influenced by their own symptoms of PTSS. Also, fathers did not report higher ratings of PTSS compared to their children. Alternatively, the discrepancy could be explained by the elicitation of the expert judgements. Especially the nurses group reported higher PTSS levels compared to the self-reports and the previously mentioned convolution of symptoms and lack of specific knowledge about PTSS might be a cause for this observation. In the recordings of the elicitation settings we found another possible cause. Five of the nurses expressed sentiments that the more severe cases came to mind more easily and therefore might be overrepresented in their beliefs. This is a clear expression of the well known availability heuristic <span class="citation">(Tversky & Kahneman, <a href="#ref-tversky_availability:_1973" role="doc-biblioref">1973</a>)</span> that can cause biases in elicitation studies <span class="citation">(O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span>. In the psychologists group only a single expert expressed a similar remark. The availability heuristic, if not remedied, might cause the discrepancy between the reference posteriors and the expert judgements.</p>
<p>The study showed that providing visual feedback on the representation of the experts’ beliefs can lead to adjustments by the experts to their input such that obvious incorrect representations of the experts’ beliefs are remedied. Unfortunately it is not possible to validate that the representation of the experts’ beliefs is actually the “true” beliefs of the expert <span class="citation">(Colson & Cooke, <a href="#ref-colson_expert_2018" role="doc-biblioref">2018</a>; O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>)</span>. However, one of the main reasons to use elicitation software is to ameliorate the effects of heuristics and biases by getting experts to actively reflect on the probability distribution that will be used to represent their beliefs. In the recordings three experts actively reflect on their distributions and adjust them based on the visual feedback and for this the purpose the elicitation software seems to have worked well. Nevertheless, it seems from our current study that even with the graphical feedback, some experts might still suffer from overconfidence, see Figure <a href="elicitlgm.html#fig:ch06fig6">6.6</a>. Expert 11, for instance, stated <em>“… of course, I have a lot of uncertainty anyway.”</em>. However, this does not seem reflected in the elicited distribution which has a 99% CI for the latent intercept [27.2, 41.7] and the latent slope [1.2, 5.9]. As the experts were only available to us for a limited time we did not provide a specialized training aimed at elicitation and overcoming heuristics associated with elicitation tasks which might be a limitation for the current study, and the associated (individual level) results.</p>
<p>This study indicated that aggregating expert judgements could potentially mitigate the severity of the individual biases, as one thereby relies less on single, possibly overconfident, experts. The aggregation of all experts’ judgements, or only the psychologists’ judgements, lead to less discrepancy between the traditionally collected data and the elicited beliefs than mostly any individual expert and the benchmarks. Aggregating or pooling of expert judgements into a single distribution is common in elicitation studies and can be done in several manners. In our current study we used opinion pooling with equal weights <span class="citation">(O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>, Chapter 9)</span>. Alternatively, there is much literature on how expert judgements could be weighted in the aggregation of views. The classical model <span class="citation">(Cooke, <a href="#ref-cooke_experts_1991" role="doc-biblioref">1991</a>, Chapter 12)</span> is one of the foremost examples of this. In the classical approach calibration questions are used to assess the experts. Based on the calibration questions experts’ judgements on the target question, or question of interest, are weighted to together form the groups weighted prior beliefs. The calibration questions should be related to the question of interest and their answers should be known but not to the experts <span class="citation">(Colson & Cooke, <a href="#ref-colson_expert_2018" role="doc-biblioref">2018</a>)</span>. It is recommended to have at least eight to ten calibration questions if dealing with continuous variables <span class="citation">(Cooke, <a href="#ref-cooke_experts_1991" role="doc-biblioref">1991</a>, Chapter 12)</span>. The experts are elicited concerning the question of interest and the calibration questions. Their answers on the calibration questions are evaluated against the known true values and the experts are rated on their informativeness and accuracy <span class="citation">(Colson & Cooke, <a href="#ref-colson_expert_2018" role="doc-biblioref">2018</a>; Cooke, <a href="#ref-cooke_experts_1991" role="doc-biblioref">1991</a>)</span>. The ratings of the weighting components is based upon the idea of KL-divergences <span class="citation">(O’Hagan et al., <a href="#ref-ohagan_uncertain_2006" role="doc-biblioref">2006</a>, Chapter 9)</span> such as we used to compare the experts’ judgements against the collected data on the question of interest directly. As far as we know there have not been any studies using the classical approach in the social sciences. Finding calibration questions turns out to be a hard problem, as knowing the true answer to these questions is required. We described the KL-divergence between the target question and the experts’ judgements, but calibrating experts based on these weight would be putting emphasis on the traditionally collected data twice. As the traditionally collected data might suffer from biases too, consider for instance the total survey error framework <span class="citation">(Groves et al., <a href="#ref-groves_survey_2011" role="doc-biblioref">2011</a>, Chapter 2)</span> including nonresponse error and measurement error, this double emphasis might not be desirable. Instead, our equal weights aggregation approach relied on the inclusion of experts with balance in views and diversity in backgrounds <span class="citation">(Cooke & Goossens, <a href="#ref-cooke_procedures_1999" role="doc-biblioref">1999</a>)</span>.</p>
<p>In conclusion, it is possible to express the experts’ domain knowledge as prior distributions using the described methodology and compare these elicited distributions to traditionally collected data. The individual expert judgements in general show quite some discrepancy in comparison to traditionally collected data, although there are notable exceptions to this. When taking the mixtures of the groups of experts the discrepancy becomes less pronounced, very much so for the psychologists group. The psychologists mixture prior has less KL-divergence than mostly any individual expert and notably less KL-divergence than benchmark 1, the uniform prior that takes the information of the measurement instrument into account. The expert judgements add information to the research area and exploring (dis)similarities between expert judgements and traditional data open up two exciting avenues for future research. First, the collection of data on the experts that might be predictive for the amount of KL-divergence they exhibit with respect to traditionally collected data. Second, the organisation of a Delphi like setting with all experts after the individual judgements are collected and compared with traditional data. The group setting can provide insights into the reasons behind the discrepancies between traditional collected data, individual experts and groups of experts. Predicting and explaining (dis)similarities between experts judgements and traditional data such as results of questionnaires can be a potential new line of research for the social sciences.</p>
<div style="page-break-after: always;"></div>
</div>
<div id="conflicts-of-interest" class="section level2 unnumbered">
<h2>Conflicts of Interest</h2>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</div>
<div id="ethics-statement" class="section level2 unnumbered">
<h2>Ethics Statement</h2>
<p>This study was carried out in accordance with the recommendations of the internal Ethics Committee of the Faculty of Social and Behavioural Sciences of Utrecht University, with written informed consent from all subjects. All subjects gave written informed consent in accordance with the Declaration of Helsinki. The protocol was approved by the internal Ethics Committee of the Faculty of Social and Behavioural Sciences of Utrecht University.</p>
</div>
<div id="acknowledgements-1" class="section level2 unnumbered">
<h2>Acknowledgements</h2>
<p>The authors are grateful to all experts for their invested time and energy.</p>
</div>
<div id="funding" class="section level2 unnumbered">
<h2>Funding</h2>
<p>This work was supported by Grant NWO-VIDI-452-14-006 from the Netherlands Organization for Scientific Research. None of the funders or sponsors of this research had any role in the design and conduct of the study; collection, management, analysis, and interpretation of data; preparation, review, or approval of the manuscript; or decision to submit the manuscript for publication.</p>
</div>
</div>
<h3>References</h3>
<div id="refs" class="references">
<div id="ref-alisic_manual_2011">
<p>Alisic, E., Eland, J., Huijbregts, R., & Kleber, R. (2011). Manual of the children’s responses to trauma inventory - revised edition.[Handleiding bij de schokverwerkingslijst voor kinderen-herziene versie]. <em>Diemen/Utrecht, the Netherlands: Institute for Psychotrauma in Collaboration with Utrecht University and University Medical Center Utrecht</em>.</p>
</div>
<div id="ref-alisic_childrens_2006">
<p>Alisic, E., Eland, J., & Kleber, R. (2006). Children’s Responses to Trauma Inventory-Revised Version [Schokverwerkingslijst Voor Kinderen-Herziene Versie]. <em>Zaltbommel/Utrecht, the Netherlands: Institute for Psychotrauma in Collaboration with Utrecht University and University Medical Center Utrecht</em>.</p>
</div>
<div id="ref-barons_eliciting_2018">
<p>Barons, M. J., Wright, S. K., & Smith, J. Q. (2018). Eliciting probabilistic judgements for integrating decision support systems. In L. C. Dias, A. Morton, & J. Quigley (Eds.), <em>Elicitation</em> (pp. 445–478). Springer.</p>
</div>
<div id="ref-bojke_eliciting_2010">
<p>Bojke, L., Claxton, K., Bravo-Vergel, Y., Sculpher, M., Palmer, S., & Abrams, K. (2010). Eliciting distributions to populate decision analytic models. <em>Value in Health</em>, <em>13</em>(5), 557–564.</p>
</div>
<div id="ref-bousquet_diagnostics_2008">
<p>Bousquet, N. (2008). Diagnostics of prior-data agreement in applied Bayesian analysis. <em>Journal of Applied Statistics</em>, <em>35</em>(9), 1011–1029.</p>
</div>
<div id="ref-buist_developmental_2002">
<p>Buist, K. L., Dekovic, M., Meeus, W., & van Aken, M. A. (2002). Developmental patterns in adolescent attachment to mother, father and sibling. <em>Journal of Youth and Adolescence</em>, <em>31</em>(3), 167–176.</p>
</div>
<div id="ref-burkner_parameterization_2019">
<p>Burkner, P.-C. (2019). Parameterization of Response Distributions in brms. Retrieved from <a href="https://cran.r-project.org/web/packages/brms/vignettes/brms_families.html">https://cran.r-project.org/web/packages/brms/vignettes/brms_families.html</a></p>
</div>
<div id="ref-catts_reading_2008">
<p>Catts, H. W., Bridges, M. S., Little, T. D., & Tomblin, J. B. (2008). Reading achievement growth in children with language impairments. <em>Journal of Speech, Language, and Hearing Research</em>.</p>
</div>
<div id="ref-R-shiny">
<p>Chang, W., Cheng, J., Allaire, J., Xie, Y., & McPherson, J. (2019). <em>Shiny: Web application framework for r</em>. Retrieved from <a href="https://CRAN.R-project.org/package=shiny">https://CRAN.R-project.org/package=shiny</a></p>
</div>
<div id="ref-colson_expert_2018">
<p>Colson, A. R., & Cooke, R. M. (2018). Expert elicitation: Using the classical model to validate experts‘ judgments. <em>Review of Environmental Economics and Policy</em>, <em>12</em>(1), 113–132.</p>
</div>
<div id="ref-cooke_experts_1991">
<p>Cooke, R. M. (1991). <em>Experts in uncertainty: Opinion and subjective probability in science</em>. Oxford University Press on Demand.</p>
</div>
<div id="ref-cooke_tu_2008">
<p>Cooke, R. M., & Goossens, L. H. J. (2008). TU Delft expert judgment data base. <em>Reliability Engineering & System Safety</em>, <em>93</em>(5), 657–674.</p>
</div>
<div id="ref-cooke_procedures_1999">
<p>Cooke, R. M., & Goossens, L. J. H. (1999). <em>Procedures guide for structured expert judgment</em>. Brussels: Commission of the European Communities.</p>
</div>
<div id="ref-dewispelare_use_1995">
<p>Dewispelare, A. R., Herren, L. T., & Clemen, R. T. (1995). The use of probability elicitation in the high-level nuclear waste regulation program. <em>International Journal of Forecasting</em>, <em>11</em>(1), 5–24.</p>
</div>
<div id="ref-dodd_global_2017">
<p>Dodd, P. J., Yuen, C. M., Sismanidis, C., Seddon, J. A., & Jenkins, H. E. (2017). The global burden of tuberculosis mortality in children: A mathematical modelling study. <em>The Lancet Global Health</em>, <em>5</em>(9), e898–e906.</p>
</div>
<div id="ref-duncan_introduction_2004">
<p>Duncan, T. E., & Duncan, S. C. (2004). An introduction to latent growth curve modeling. <em>Behavior Therapy</em>, <em>35</em>(2), 333–363.</p>
</div>
<div id="ref-egberts_mother_2018">
<p>Egberts, M. R., van de Schoot, R., Geenen, R., & van Loey, N. E. (2018). Mother, father and child traumatic stress reactions after paediatric burn: Within-family co-occurrence and parent-child discrepancies in appraisals of child stress. <em>Burns</em>, <em>44</em>(4), 861–869.</p>
</div>
<div id="ref-elfadaly_eliciting_2017">
<p>Elfadaly, F. G., & Garthwaite, P. H. (2017). Eliciting Dirichlet and Gaussian copula prior distributions for multinomial models. <em>Statistics and Computing</em>, <em>27</em>(2), 449–467.</p>
</div>
<div id="ref-fischer_estimating_2013">
<p>Fischer, K., Lewandowski, D., & Janssen, M. (2013). Estimating unknown parameters in haemophilia using expert judgement elicitation. <em>Haemophilia</em>, <em>19</em>(5), e282–e288.</p>
</div>
<div id="ref-fisher_software_2012">
<p>Fisher, R., O’Leary, R. A., Low-Choy, S., Mengersen, K., & Caley, M. J. (2012). A software tool for elicitation of expert knowledge about species richness or similar counts. <em>Environmental Modelling & Software</em>, <em>30</em>, 1–14.</p>
</div>
<div id="ref-garthwaite_prior_2013">
<p>Garthwaite, P. H., Al-Awadhi, S. A., Elfadaly, F. G., & Jenkinson, D. J. (2013). Prior distribution elicitation for generalized linear and piecewise-linear models. <em>Journal of Applied Statistics</em>, <em>40</em>(1), 59–75.</p>
</div>
<div id="ref-goldstein_lay_2014">
<p>Goldstein, D. G., & Rothschild, D. (2014). Lay understanding of probability distributions. <em>Judgment & Decision Making</em>, <em>9</em>(1).</p>
</div>
<div id="ref-gosling_shelf:_2018">
<p>Gosling, J. P. (2018). SHELF: The Sheffield elicitation framework. In <em>Elicitation</em> (pp. 61–93). Springer.</p>
</div>
<div id="ref-gronau_informed_2019">
<p>Gronau, Q. F., Ly, A., & Wagenmakers, E.-J. (2019). Informed Bayesian t-tests. <em>The American Statistician</em>, 1–14.</p>
</div>
<div id="ref-groves_survey_2011">
<p>Groves, R. M., Fowler Jr, F. J., Couper, M. P., Lepkowski, J. M., Singer, E., & Tourangeau, R. (2011). <em>Survey methodology</em> (Vol. 561). John Wiley & Sons.</p>
</div>
<div id="ref-haakma_belief_2014">
<p>Haakma, W., Steuten, L. M., Bojke, L., & IJzerman, M. J. (2014). Belief elicitation to populate health economic models of medical diagnostic devices in development. <em>Applied Health Economics and Health Policy</em>, <em>12</em>(3), 327–334.</p>
</div>
<div id="ref-hald_world_2016">
<p>Hald, T., Aspinall, W., Devleesschauwer, B., Cooke, R., Corrigan, T., Havelaar, A. H., … Angulo, F. J. (2016). World Health Organization estimates of the relative contributions of food to the burden of disease due to selected foodborne hazards: A structured expert elicitation. <em>PloS One</em>, <em>11</em>(1), e0145839.</p>
</div>
<div id="ref-hampson_bayesian_2014">
<p>Hampson, L. V., Whitehead, J., Eleftheriou, D., & Brogan, P. (2014). Bayesian methods for the design and interpretation of clinical trials in very rare diseases. <em>Statistics in Medicine</em>, <em>33</em>(24), 4186–4201.</p>
</div>
<div id="ref-hampson_elicitation_2015">
<p>Hampson, L. V., Whitehead, J., Eleftheriou, D., Tudur-Smith, C., Jones, R., Jayne, D., … Caldas, A. (2015). Elicitation of expert prior opinion: Application to the MYPAN trial in childhood polyarteritis nodosa. <em>PLoS One</em>, <em>10</em>(3), e0120981.</p>
</div>
<div id="ref-ho_volcanic_1997">
<p>Ho, C.-H., & Smith, E. I. (1997). Volcanic hazard assessment incorporating expert knowledge: Application to the Yucca Mountain region, Nevada, USA. <em>Mathematical Geology</em>, <em>29</em>(5), 615–627.</p>
</div>
<div id="ref-james_elicitator:_2010">
<p>James, A., Choy, S. L., & Mengersen, K. (2010). Elicitator: An expert elicitation tool for regression in ecology. <em>Environmental Modelling & Software</em>, <em>25</em>(1), 129–145.</p>
</div>
<div id="ref-kullback_information_1951">
<p>Kullback, S., & Leibler, R. A. (1951). On information and sufficiency. <em>The Annals of Mathematical Statistics</em>, <em>22</em>(1), 79–86.</p>
</div>
<div id="ref-lek_development_2018">
<p>Lek, K., & van de Schoot, R. (2018). Development and evaluation of a digital expert elicitation method aimed at fostering elementary school teachers’ diagnostic competence. In (Vol. 3, p. 82). Frontiers.</p>
</div>
<div id="ref-lek_how_2019">
<p>Lek, K., & van de Schoot, R. (2019). How the Choice of Distance Measure Influences the Detection of Prior-Data Conflict. <em>Entropy</em>, <em>21</em>(5), 446.</p>
</div>
<div id="ref-little_longitudinal_2013">
<p>Little, T. D. (2013). <em>Longitudinal structural equation modeling</em>. Guilford press.</p>
</div>
<div id="ref-little_methods_2006">
<p>Little, T. D., Bovaird, J. A., & Slegers, D. W. (2006). Methods for the analysis of change. <em>Handbook of Personality Development</em>, 181–211.</p>
</div>
<div id="ref-low-choy_elicitator:_2012">
<p>Low-Choy, S., James, A., Murray, J., & Mengersen, K. (2012). Elicitator: A user-friendly, interactive tool to support scenario-based elicitation of expert knowledge. In <em>Expert knowledge and its application in landscape ecology</em> (pp. 39–67). Springer.</p>
</div>
<div id="ref-morris_web-based_2014">
<p>Morris, D. E., Oakley, J. E., & Crowe, J. A. (2014). A web-based tool for eliciting probability distributions from experts. <em>Environmental Modelling & Software</em>, <em>52</em>, 1–4.</p>
</div>
<div id="ref-murphy_subjective_1974">
<p>Murphy, A. H., & Winkler, R. L. (1974). Subjective probability forecasting experiments in meteorology: Some preliminary results. <em>Bulletin of the American Meteorological Society</em>, <em>55</em>(10), 1206–1216.</p>
</div>
<div id="ref-murphy_probability_1984">
<p>Murphy, A. H., & Winkler, R. L. (1984). Probability forecasting in meteorology. <em>Journal of the American Statistical Association</em>, <em>79</em>(387), 489–500.</p>
</div>
<div id="ref-R-SHELF">
<p>Oakley, J. (2019). <em>SHELF: Tools to support the sheffield elicitation framework</em>. Retrieved from <a href="https://CRAN.R-project.org/package=SHELF">https://CRAN.R-project.org/package=SHELF</a></p>
</div>
<div id="ref-ohagan_uncertain_2006">
<p>O’Hagan, A., Buck, C. E., Daneshkhah, A., Eiser, J. R., Garthwaite, P. H., Jenkinson, D. J., … Rakow, T. (2006). <em>Uncertain judgements: Eliciting experts’ probabilities</em>. John Wiley & Sons.</p>
</div>
<div id="ref-orth_life-span_2012">
<p>Orth, U., Robins, R. W., & Widaman, K. F. (2012). Life-span development of self-esteem and its effects on important life outcomes. <em>Journal of Personality and Social Psychology</em>, <em>102</em>(6), 1271.</p>
</div>
<div id="ref-runge_interactive_2013">
<p>Runge, A. K., Scherbaum, F., Curtis, A., & Riggelsen, C. (2013). An interactive tool for the elicitation of subjective probabilities in probabilistic seismic‐hazard analysis. <em>Bulletin of the Seismological Society of America</em>, <em>103</em>(5), 2862–2874.</p>
</div>
<div id="ref-truong_web-based_2013">
<p>Truong, P. N., Heuvelink, G. B., & Gosling, J. P. (2013). Web-based tool for expert elicitation of the variogram. <em>Computers & Geosciences</em>, <em>51</em>, 390–399.</p>
</div>
<div id="ref-tversky_availability:_1973">
<p>Tversky, A., & Kahneman, D. (1973). Availability: A heuristic for judging frequency and probability. <em>Cognitive Psychology</em>, <em>5</em>(2), 207–232.</p>
</div>
<div id="ref-van_de_schoot_bayesian_2018">
<p>van de Schoot, R., Sijbrandij, M., Depaoli, S., Winter, S. D., Olff, M., & van Loey, N. E. (2018). Bayesian PTSD-trajectory analysis with informed priors based on a systematic literature search and expert elicitation. <em>Multivariate Behavioral Research</em>, <em>53</em>(2), 267–291.</p>
</div>
<div id="ref-van_de_schoot_systematic_2017">
<p>van de Schoot, R., Winter, S. D., Ryan, O., Zondervan-Zwijnenburg, M., & Depaoli, S. (2017). A systematic review of Bayesian articles in psychology: The last 25 years. <em>Psychological Methods</em>, <em>22</em>(2), 217.</p>
</div>
<div id="ref-veen_using_2018">
<p>Veen, D., Stoel, D., Schalken, N., Mulder, K., & van de Schoot, R. (2018). Using the Data Agreement Criterion to Rank Experts’ Beliefs. <em>Entropy</em>, <em>20</em>(8), 592.</p>
</div>
<div id="ref-veen_proposal_2017">
<p>Veen, D., Stoel, D., Zondervan-Zwijnenburg, M., & van de Schoot, R. (2017). Proposal for a Five-Step Method to Elicit Expert Judgement. <em>Frontiers in Psychology</em>, <em>8</em>, 2110.</p>
</div>
<div id="ref-zondervan-zwijnenburg_application_2017">
<p>Zondervan-Zwijnenburg, M., van de Schoot-Hubeek, W., Lek, K., Hoijtink, H., & van de Schoot, R. (2017b). Application and evaluation of an expert judgment elicitation procedure for correlations. <em>Frontiers in Psychology</em>, <em>8</em>, 90.</p>
</div>
</div>
<div class="footnotes">
<hr />
<ol start="2">
<li id="fn2"><p>Note that the mixtures are based on normal approximations of the elicited skewed normal distributions due to computational instability of the mixture distributions when skewed normal expert priors where used.<a href="elicitlgm.html#fnref2" class="footnote-back">↩</a></p></li>
</ol>
</div>
</section>
</div>
</div>
</div>
<a href="Burns.html" class="navigation navigation-prev " aria-label="Previous page"><i class="fa fa-angle-left"></i></a>
<a href="thesisdiscussion.html" class="navigation navigation-next " aria-label="Next page"><i class="fa fa-angle-right"></i></a>
</div>
</div>
<script src="libs/gitbook-2.6.7/js/app.min.js"></script>
<script src="libs/gitbook-2.6.7/js/lunr.js"></script>
<script src="libs/gitbook-2.6.7/js/clipboard.min.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-search.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-sharing.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-fontsettings.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-bookdown.js"></script>
<script src="libs/gitbook-2.6.7/js/jquery.highlight.js"></script>
<script src="libs/gitbook-2.6.7/js/plugin-clipboard.js"></script>
<script>
gitbook.require(["gitbook"], function(gitbook) {
gitbook.start({
"sharing": {
"github": false,
"facebook": true,
"twitter": true,
"google": false,
"linkedin": true,
"weibo": false,
"instapaper": false,
"vk": false,
"all": ["facebook", "google", "twitter", "linkedin", "weibo", "instapaper"]
},
"fontsettings": {
"theme": "white",
"family": "sans",
"size": 2
},
"edit": {
"link": null,
"text": null
},
"history": {
"link": null,
"text": null
},
"download": ["Dissertation_Duco_Veen.pdf"],
"toc": {
"collapse": "section"
},
"search": true
});
});
</script>
<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
(function () {
var script = document.createElement("script");
script.type = "text/javascript";
var src = "true";
if (src === "" || src === "true") src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-MML-AM_CHTML";
if (location.protocol !== "file:")
if (/^https?:/.test(src))
src = src.replace(/^https?:/, '');
script.src = src;
document.getElementsByTagName("head")[0].appendChild(script);
})();
</script>
</body>
</html>