-
Notifications
You must be signed in to change notification settings - Fork 2
/
index.html
10401 lines (10342 loc) · 390 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="generator" content="pandoc">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<meta name="author" content="Dave Kinkead">
<title>How does school choice effect measures of school performance?</title>
<style type="text/css">code{white-space: pre;}</style>
<!--[if lt IE 9]>
<script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
<![endif]-->
<style type="text/css">
* { margin: 0; padding: 0; font-size: 20px; }
@font-face {
font-family: 'Vollkorn';
font-style: normal;
font-weight: 400;
src: local('Vollkorn Regular'), local('Vollkorn-Regular'), url(///fonts.gstatic.com/s/vollkorn/v6/idGKtgpe38okB6bfeHMsLHYhjbSpvc47ee6xR_80Hnw.woff2) format('woff2'), url(http://fonts.gstatic.com/s/vollkorn/v6/BCFBp4rt5gxxFrX6F12DKnYhjbSpvc47ee6xR_80Hnw.woff) format('woff');
}
@font-face {
font-family: 'Vollkorn';
font-style: normal;
font-weight: 700;
src: local('Vollkorn Bold'), local('Vollkorn-Bold'), url(///fonts.gstatic.com/s/vollkorn/v6/wMZpbUtcCo9GUabw9JODeogp9Q8gbYrhqGlRav_IXfk.woff2) format('woff2'), url(http://fonts.gstatic.com/s/vollkorn/v6/wMZpbUtcCo9GUabw9JODerrIa-7acMAeDBVuclsi6Gc.woff) format('woff');
}
@font-face {
font-family: 'Vollkorn';
font-style: italic;
font-weight: 400;
src: local('Vollkorn Italic'), local('Vollkorn-Italic'), url(///fonts.gstatic.com/s/vollkorn/v6/Oiz0tNwvC-Nd29SBQWfWTAzyDMXhdD8sAj6OAJTFsBI.woff2) format('woff2'), url(http://fonts.gstatic.com/s/vollkorn/v6/Oiz0tNwvC-Nd29SBQWfWTL3hpw3pgy2gAi-Ip7WPMi0.woff) format('woff');
}
@font-face {
font-family: 'Vollkorn';
font-style: italic;
font-weight: 700;
src: local('Vollkorn Bold Italic'), local('Vollkorn-BoldItalic'), url(///fonts.gstatic.com/s/vollkorn/v6/KNiAlx6phRqXCwnZZG51JGaVI6zN22yiurzcBKxPjFE.woff2) format('woff2'), url(http://fonts.gstatic.com/s/vollkorn/v6/KNiAlx6phRqXCwnZZG51JHbFhgvWbfSbdVg11QabG8w.woff) format('woff');
}
html {
font-family: "Vollkorn", sans-serif;
font-weight: normal;
margin: 0;
padding: 0;
color: #333;
}
h1, h2, h3, h4, h5, h6, p, ul, pre, figcaption, .footnotes {
padding: 1rem;
max-width: 900px;
margin: 0 auto;
line-height: 1.67;
}
h1 {
font-size: 2rem;
text-transform: capitalize;
line-height: 1.33;
}
h2 {
font-size: 1.5rem;
}
h3 {
font-size: 1.25rem;
}
h1, h2, h3 {
margin: 2rem auto;
text-align: center;
}
h5 {
font-size: 1rem;
font-weight: normal;
text-align: center;
}
blockquote {
font-style: italic;
padding: 0 1rem;
}
pre {
margin-top: 0.5rem;
margin-bottom: 0.75rem;
}
code {
font-family: "Courier New", monospace;
font-size: 0.9rem;
}
p code {
background-color: #EFEFEF;
padding: 0.1rem 0.25rem;
}
pre code {
display: block;
padding: 0 1rem;
line-height: 1.1rem;
overflow: scroll;
max-width: 95%;
}
figcaption {
text-align: center;
padding-bottom: 2rem;
}
sup, sub, a.footnote {
font-size: 1.4ex;
height: 0;
line-height: 1;
position: relative;
vertical-align: super;
}
hr {
margin: 4em auto;
max-width: 700px;
}
p.author {
text-align: center;
}
p.author:before {
content: "by ";
}
.footnotes {
/*list-style-position: outside;*/
}
.footnotes li {
margin: 1rem 2rem;
}
@media print {
* {
font-size: 20px;
}
body {
margin: 0.5cm;
text-align: justify;
}
h1, h2, h3, p {
font-family: "Times New Roman", serif;
margin: 1rem auto;
}
svg {
max-width: 100vh;
margin-top: -1rem;
}
figcaption {
margin-top: -1rem;
}
.simulation {
page-break-inside: avoid;
}
}</style>
</head>
<body>
<header>
<h1 class="title">How does school choice effect measures of school performance?</h1>
<p class="author">Dave Kinkead</p>
</header>
<h5 id="this-paper-is-designed-to-be-viewed-interactively-and-is-best-viewed-in-html-at-httpsdave.kinkead.com.auschool-performance.-a-static-pdf-version-is-also-provided.">This paper is designed to be viewed interactively and is best viewed in HTML at <a href="https://dave.kinkead.com.au/school-performance" class="uri">https://dave.kinkead.com.au/school-performance</a>. A static PDF version is also provided.</h5>
<h2 id="abstract">Abstract</h2>
<blockquote>
<p>Measuring school performance has become an important activity within our education systems. Today, most developed countries have some metric to rank and compare schools — Adequate Yearly Progress (AYP) of No Child Left Behind (NCLB), Progress 8 scores, National Assessment Program - Literacy and Numeracy (NAPLAN), Trends in International Mathematics and Science Study (TIMSS), and the Programme for International Student Assessment (PISA).</p>
</blockquote>
<blockquote>
<p>Yet these measures are only substitutes for what we are really interested in — how much do schools influence student achievement. Because we can’t measure the causal impact of schooling on student achievement directly, we can’t know what impact schools have with any certainty. Instead, we must <em>infer</em> the causal impact of schooling on student achievement by way of proxy measures such as standardised student test scores which presents us with a significant epistemic challenge.</p>
</blockquote>
<blockquote>
<p>This paper introduces a novel approach to computer simulation to argue that the presence of choice or selectivity within a school system undermines measures of school performance based on student acheivement. In it, I develop a simple model in order to investigate the robustness of the inference from student achievement to school performance. I then demonstrate how the introduction of school choice or selectivity leads to a complete failure of the inference mechanism.</p>
</blockquote>
<blockquote>
<p>My claim is simple — <em>if an inference is unreliable in an ideal setting, then it will certainly be unreliable in a non-ideal one</em>. By eliminating measurement errors and abstracting away confounders, computer simulation allows us to create the ideal conditions from which to assess the quality of this inference. Thus, if the conditions in this model are present in the real world, then we should be very sceptical inferring anything about school performance from student achievement.</p>
</blockquote>
<h2 id="introduction">Introduction</h2>
<p>There are a variety of ways we might measure school performance. One way is to <em>measure the things schools do</em> — the practices, pedagogies, and processes they apply. If their application of a process is faithful to some standard, then we might judge them to be performing well. If not, then we judge them poorly. But a more common approach is to <em>measure the things schools produce</em> —- how many students qualified for university, how many found well paying jobs, or especially, how well students performed in tests.</p>
<p>Measuring educational outputs is much simpler than measuring processes. However we measure the academic achievement of students — whether by <em>declarative knowledge</em>, the learning and appropriate recall of particular facts; or by <em>procedural knowledge</em>, the application of skill and know-how — we have an abundance of existing data available to us. The tools needed to obtain this data are also relatively easy to administer — standardised literacy and numeracy tests like NAPLAN <span class="citation" data-cites="naplan2015">(NAPLAN, <a href="#ref-naplan2015">2015</a>)</span>; the Peabody Picture Vocabulary Test <span class="citation" data-cites="dunn1965">(L. M. Dunn, Dunn, Bulheller, & Häcker, <a href="#ref-dunn1965">1965</a>)</span>; IQ scores or the Wechsler Intelligence Scale for Children (WISC-IV) <span class="citation" data-cites="wechsler2003">(Wechsler, <a href="#ref-wechsler2003">2003</a>)</span>. Yet the metrics from these tools are just proxies for what we really want to know — what is the educational impact of schooling upon students?</p>
<p>Accurately measuring school performance by proxy however, is hard. The challenges associated with inferring school performance from student achievement have long been documented. Ecological models, for example, stress the important role non-school factors like family and neighbourhood have on student performance <span class="citation" data-cites="bronfenbrenner1994">(Bronfenbrenner, <a href="#ref-bronfenbrenner1994">1994</a>)</span>, <span class="citation" data-cites="zubrick2000">(Zubrick, Silburn, Williams, & Vimpani, <a href="#ref-zubrick2000">2000</a>)</span>. Further clouding our understanding are the often competing causal theories that attempt to explain how schools affect student achievement - peer effect <span class="citation" data-cites="hanushek2003">(Hanushek, Kain, Markman, & Rivkin, <a href="#ref-hanushek2003">2003</a>)</span>, class size policies <span class="citation" data-cites="ehrenberg2001">(Ehrenberg, Brewer, Gamoran, & Willms, <a href="#ref-ehrenberg2001">2001</a>)</span>, or teacher training and qualifications <span class="citation" data-cites="kosgei2013">(Kosgei, Mise, Odera, & Ayugi, <a href="#ref-kosgei2013">2013</a>)</span>. How and to what degree schools themselves impact student achievement remains an open question. How much the changes in student achievement can be attributed to school performance is, therefore, also uncertain.</p>
<p>Measuring school performance is hard because measuring causation is hard. The limits of causal knowledge have been well documented in science and philosophy (see <span class="citation" data-cites="schaffer2016">Schaffer (<a href="#ref-schaffer2016">2016</a>)</span>, <span class="citation" data-cites="hitchcock2019">Hitchcock (<a href="#ref-hitchcock2019">2019</a>)</span>). Causation cannot be observed directly — it can’t be seen, heard, or touched. Neither can it be known <em>a priori</em>. Instead, causal connections must be <em>inferred</em> from their observable, posited effects <span class="citation" data-cites="hume1748">(Hume, <a href="#ref-hume1748">1748</a>)</span>. In order to discover the causes of effects, we try to hold all but a few variables fixed, and observe the covariance between them in order to identify causality.</p>
<p>In complex systems, accurate causal inferences are especially challenging. Common causes, feedback loops, under-determination, over-determination, and causal indeterminacy all strain the certainty of our inference mechanisms despite the best controls, protocols, and experimental design we might put in place. Causal inference is difficult. Inferring the causal impact of a school on student achievement in a complex education system with multiple confounders is especially difficult. Yet infer it we must. So an important question we need to ask is <em>just how warranted is this inference from student achievement to school performance</em>?</p>
<p>In some scenarios, inferring school performance from student achievement might be perfectly justified. Changes in student achievement might be largely or even wholly explainable by a school’s causal impact. In these instances, when students results improve, we can justifiably say that because the school’s causal impact is largely responsible, the school performance has improved. In many other scenarios however, we might have serious grounds for scepticism. Confounders such as parental age and socio-economic status <span class="citation" data-cites="caro2009">(Caro, McDonald, & Willms, <a href="#ref-caro2009">2009</a>)</span>, birth weight, neighbourhood characteristics <span class="citation" data-cites="nghiem2015">(Nghiem, Nguyen, Khanam, & Connelly, <a href="#ref-nghiem2015">2015</a>)</span>, or even a student’s breakfast consumption <span class="citation" data-cites="adolphus2013">(Adolphus, Lawton, & Dye, <a href="#ref-adolphus2013">2013</a>)</span> might explain a great deal about differences in student achievement. In these cases, when the extent of a school’s causal impact is uncertain, then we should question whether we can infer anything at all about school performance from changes in student achievement.</p>
<p>If inferring from observable effects to their causes is difficult, then judging the quality of the inference mechanism is even harder. Doing so requires some standard against which we can make comparisons but our knowledge of this standard is limited by the same problem of inference. If we could somehow know independently the causal impact of schools upon students, then we could judge the qualify of our inference from empiric data. But how can we judge the quality of the inference from student achievement to school performance if we can’t be certain what is the cause of student achievement in the first place? We lack the epistemological foundations to properly ground our second order judgements.</p>
<p>This is a problem that empiric approaches cannot overcome. If we are to use empiric data from student achievement to make causal inferences about school performance, then we can’t use that same data to judge the quality of the causal inference mechanism. If however, we are to use the same empiric data to judge the quality of the inference, then we need to already know the cause — but if we already knew the cause, we wouldn’t need to infer school performance via the proxy of student achievement in the first place. We need another way.</p>
<h2 id="methodology">Methodology</h2>
<p>Computer simulation offers us one way out of this problem. Simulation allows us to not only model how we think the world <em>is</em>, but to also specify how we think the world <em>should be</em>. It allows us to know the causal relationships <em>within</em> the model with certainty because they are explicitly stipulated in code <em>ex ante</em>. With causality known by stipulation <em>a priori</em>, we can then observe the empiric data the simulation generates <em>a posteriori</em> and assess the quality of the inference mechanism. We can judge how good the inference mechanism from student achievement to school performance really is - in our model at least.</p>
<p>The purpose of this simulation is therefore not to explain or predict the real world but to create a yard stick by which we can judge our causal inferences in the real world. The model simplifies and abstracts away all possible confounding causes of student performance. Importantly, it eliminates any errors when measuring student performance and stipulates that schooling is the <em>only</em> cause of changes in student performance.<a href="#fn1" class="footnoteRef" id="fnref1"><sup>1</sup></a> This creates ideal conditions for assessing the quality of our second order judgements — the inference from student achievement to school performances.</p>
<p>Because the causal impact of a school on student achievement in the simulation is known by stipulation, we can assess how accurately the empiric data about student achievement generated from the simulation maps to the stipulated causal impact of the school — to our belief about school performance. If the inference mechanism from student acheivement to school performance is warranted in the simulation when the causal mechanism is known by stipulation, then we have some grounds to be confident about the inference mechanism in the real world, additional confounders not withstanding. <em>But if the inference isn’t warranted when the causal mechanism is known, then it can’t be warranted when the causal mechanism is unknown.</em> This simulation will therefore allow us to say when the student-acheivement-to-school-performance inference <em>might</em> be warranted, and when it <em>cannot be</em> warranted. It will give us a yard stick against which we can judge an inference as <em>possibly</em> or <em>definately not</em> warranted.</p>
<p>The methodological approach of this article is different from typical philosophical fare, even philosophical work involving computer simulation. Written in <a href="http://Coffeescript.org/">Literate Coffeescript</a>, this article is simultaneously a philosophical argument and a computer simulation that demonstrates the claims of the argument from within it. Literate Programming <span class="citation" data-cites="knuth1984">(Knuth, <a href="#ref-knuth1984">1984</a>)</span> involves embedding computer code within a written argument and has a great deal to offer scholarly writing. The compter code and written argument are one.</p>
<p>Firstly, it ensures that all assumptions of the model are explicit. Computer programs are deterministic, so all the instructions necessary for the simulation to run have to be explicitly documented. Secondly, it allays concerns often directed at simulation — the challenges of validation and replication. Often, simulations are ‘black boxes’ of code, opaque to the reader and reviewer alike. Because all the code necessary for the running simulation is embedded in the article, replication is as simple as following the installation instructions in the appendix.</p>
<p>Lastly, it enhances the persuasiveness of argument. Compared to traditional static argument, the dynamic simulation offered in this paper allows the reader to experience the argument evolve from intial definitions to final conclusion as more premises are added. This is especially useful when dealing with counter-intuitive results that might require considerable concentration and reflection to accept.<a href="#fn2" class="footnoteRef" id="fnref2"><sup>2</sup></a></p>
<p>I make no assumptions about the reader’s knowledge of computer programming. Simulation code is indicated by <code>monospace font and indented blocks</code>, is written to be as informative as possible, and is fully described in the surrounding text. Any code that is not germane to the argument itself has been placed in the appendix. Coffeescript is used because it runs in any modern web browser and has a syntax close to natural language. Best viewed in HTML at <a href="http://blind-review.github.io/school-performance/" class="uri">http://blind-review.github.io/school-performance/</a> to take advantage of the interactive visualisations, a static but less engaging version of this article is available in PDF or print.</p>
<h2 id="a-model-of-school-performance">A Model of School Performance</h2>
<p>We begin by defining a model of students. Students have a level of academic achievement measured from a minimum of <code>0.0</code> to a maximum of <code>1.0</code>. Academic achievement is broadly construed and can be interpreted as any attribute of value that can be influenced by schooling. This achievement is randomly generated with a uniform distribution and is centred around a mean of <code>0.5</code>.</p>
<pre><code>class Student
constructor: () ->
@achievement = Math.random()</code></pre>
<p>Next we model schools. Schools are collections of students upon whose acheivement they causally impact. In short, <em>schools teach students</em>. The causal impact of a school on a student’s academic performance ranges from <code>-1.0</code> (a strong negative impact) to <code>1.0</code> (a strong positive impact). This impact is the same for all students and is fixed for the duration of the simulation. Again, no claim is made about how educational impact is transmitted to student achievement in the real world. Policy, pedagogy, or the school environment are all possible causal mechanisms compatible with this model. The key stipulation is that <code>impact</code> is the <em>only</em> causal mechanism at work.</p>
<pre><code>class School
constructor: (@id, @impact) -></code></pre>
<p>Finally, we create a class to encapsulate the simulation itself. We will construct each simulations with a profile describing the desired attributes of the schools and student distribution. We then create 1000 students of random achievement and assign them in the schools according to the stipulated profile.</p>
<pre><code>class Simulation
constructor: (@profile) ->
@schools = for school, id in @profile.schools
new School(id, school.impact)
@students = [1..1000].map () =>
student = new Student()
student.school = randomly_assign(student, @schools, @profile.skew)
student</code></pre>
<p>The initial assignment of students between schools is determined by a skew factor ranging from <code>0.0</code> (all students of below average achievement are in the first school) to <code>1.0</code> (all students of above average achievement students are in the first school). The default value of <code>0.5</code> results in an even distribution of students by achievement. Subsequent enrolments will be random.</p>
<pre><code>randomly_assign = (student, schools, skew=0.5) ->
[first, second] = if student.achievement > 0.5 then [0,1] else [1,0]
if Math.random() < skew then schools[first] else schools[second]</code></pre>
<p>Events in the simulation will occur during a generic time period called a <code>tick</code>. A tick can represent any fixed period of time such as a term, semester, or year. During each tick schools will <code>teach</code> students, some students will <code>graduate</code>, and some new students will <code>enrol</code>.</p>
<pre><code>Simulation::tick = () ->
@teach()
@graduate()
@enrol()</code></pre>
<p>Schools causally impact a student’s achievement — schools teach and students learn. Again, the actual causal mechanism the model describes could be a variety of influences such as pedagogy, policy, or classroom environment. By stipulation however, schooling is the <em>only</em> mechanism by which student achievement can change within the model. The causal impact on student achievement is assumed to be linear and a constraint is applied to ensure achievement remains between <code>0.0</code> and <code>1.0</code>.</p>
<pre><code>Simulation::teach = () ->
transference=0.2
@students.forEach (student) ->
student.achievement = student.achievement * (student.school.impact * transference + 1)
student.achievement = Math.min student.achievement, 1.0</code></pre>
<p>During each tick, a percentage of students will graduate and be replaced by new students enrolling. We will set the graduation rate at 20% per tick.</p>
<pre><code>Simulation::graduate = (graduation_rate=0.2) ->
@graduates = @students.filter (student) ->
Math.random() < graduation_rate</code></pre>
<p>Identifying the ‘best’ school requires us to rank schools based on student performance so we create a simple league table that all students have knowledge of.</p>
<pre><code>Simulation::rank_schools = () ->
@measure_schools()
[best, worst] = if @schools[0].score > @schools[1].score
then [@schools[0], @schools[1]]
else [@schools[1], @schools[0]]</code></pre>
<p>Measuring school performance is then done by calculating the average student achievement in each school. This acts as an error free method of student assessment which removes concerns about measurement validation from the model. School performance will therefore perfectly track the academic achievement of the students enrolled.</p>
<pre><code>Simulation::measure_schools = () ->
@schools.map (school) =>
students = @students.filter (student) ->
student.school.id is school.id
total = students.map (student) ->
student.achievement
.reduce (prev, curr) ->
prev + curr
school.score = total / students.length</code></pre>
<p>Finally, we will introduce an optional <em>selectivity</em> parameter. If present, students will prefer to enroll in the best performing school, and schools will prefer to admit the best achieving students, so that all schools have roughly even numbers of students.</p>
<pre><code>Simulation::enrol = () ->
[best, worst] = @rank_schools()
@graduates.forEach (student) =>
student.achievement = Math.random()
if Math.random() < @profile.selectivity
student.school = if student.achievement > 0.5
then @schools[best.id]
else @schools[worst.id] </code></pre>
<p>We now have a complete model of students with initial uniformly random <code>achievement</code>; schools whose stipulated <code>impact</code> is the only mechanism for affecting student achievement; a perfect measure of school performance as average student achievement; methods for some student to graduate and new ones to enrol; and a means of varying how much <em>selectivity</em> is present in the school system as well as the initial <em>skew</em> of above average students. In short, we have created perfect conditions for measuring a school’s causal impact in order judge the quality of the inference from student achievement to school performance.</p>
<h2 id="simulations">Simulations</h2>
<p>With the model defined, we begin with a test to ensure the code is working as expected and ground our confidence in the simulation. We will begin with students split evenly between the two schools. We stipulate that schools have no causal impact on student achievement, and that there is no selectivity within the system. Because enrolment is completely random and there is no causal impact from schools, what we <em>should</em> observe is no significant change within and between schools.</p>
<p>The movement in the simulation visually represents a <code>tick</code>. Student achievement is represented in colour - blue for > <code>0.5</code> and red for < <code>0.5</code> achievement. The profile of the simulation is expressed in the code below and the simulation can be started or stopped by clicking on it.</p>
<pre><code>sanity_check_1 = {
schools: [{impact: 0.0}, {impact: 0.0}],
selectivity: 0.0
}</code></pre>
<figure class="simulation">
<div id="sanity-check-1">
</div>
<figcaption>
Simulation 1: No impact, no selectivity.
</figcaption>
</figure>
<p>Running the simulation generates the expected results. During each tick, the <code>teach</code> method has no impact on student achievement because both schools have <code>0.0</code> impact. Some students graduate and are replaced with new students of random achievement. As such, school performance, measured by average student achievement, remains close to 0.5.</p>
<p>Next, we will model two schools with different causal impact - one positive and one negative - but again with no selectivity. While both schools will start with similar student achievement levels, the <code>teach</code> method <em>should</em> see student achievement in the first school increase while it decreases in the second.</p>
<pre><code>sanity_check_2 = {
schools: [{impact: 0.5}, {impact: -0.5}],
selectivity: 0.0
}</code></pre>
<figure class="simulation">
<div id="sanity-check-2">
</div>
<figcaption>
Simulation 2: School impact, no selectivity.
</figcaption>
</figure>
<p>Again, the simulation generates the expected results with one school’s performance stabilising at approximately <code>0.66</code> and the other at <code>0.33</code>. In both cases, the simulation performed as expected and our inference from student achievement to school performance is warranted. In these scenarios we can reliably infer school performance from student achievement.</p>
<h2 id="school-performance-as-shifting-averages">School Performance as Shifting Averages</h2>
<p>Most schools systems have some degree of selectivity. This might be selectivity on the part of the school where only the top achieving students are admitted; or it might be selectivity on the part of the students or parents who explicity choose one school over another. Selectivity can also be implicit when for example, parents choose a ‘good’ neighbourhood with ‘good’ schools to live in.</p>
<p>In the next scenario, we take parameters from Simulation 1 where schools have no causal impact on student achievement but introduce selectivity into the model. Because both schools are identical in terms of their causal impact, and school <code>impact</code> is the only causal mechanism in the model, <em>any changes in school performance must be the result of selectivity</em>.</p>
<pre><code>shifting_averages = {
schools: [{impact: 0.0}, {impact: 0.0}],
selectivity: 0.5
}</code></pre>
<figure class="simulation">
<div id="shifting-averages">
</div>
<figcaption>
Simulation 3: No impact but with selectivity.
</figcaption>
</figure>
<p>While the performance of both schools is approximately <code>0.5</code> when the simulation begins, any minor imbalance in relative performance caused by random variation in new student achievement results in a run-away effect. As soon as one school is perceived to perform better than others, <em>selectivity</em> ensures that students who do selectively enrol, choose the school with the higher percentage of high achievement students. After a few ticks, the simulation stabilises with one school performing at approximate <code>0.63</code> and the other at <code>0.37</code>.</p>
<p>Recall however that the impact of schools is stipulated as zero - both schools are identical. Individual student achievement never changes because schools have no impact in this scenario. The large differences in school performance are therefore completely explained by <em>selectivity</em>. As selectivity and the percentage of students enrolling non-randomly increases, so too does the performance difference. Differences in school performance as measured by student achievement are the result of nothing more than shifting averages of student achievement. When selectivity is present, the inference from student achievement to school performance is unreliable.</p>
<h2 id="performance-is-relative">Performance is Relative</h2>
<p>The previous simulation showed how significant performance differences can arise even when schools are causally identical. The next simulations demonstrate how a school with a low or negative impact can appear to perform well above what it should. We begin with two schools having differing levels of negative impact but with no selectivity. In this case the average performance of both schools <em>should</em> decrease, propped up only by new enrolments whose average achievement is <code>0.5</code>.</p>
<pre><code>relative_1 = {
schools: [{impact: -0.25}, {impact: -0.5}],
selectivity: 0.0
}</code></pre>
<figure class="simulation">
<div id="relative-1">
</div>
<figcaption>
Simulation 4: Negative impact, no selectivity.
</figcaption>
</figure>
<p>Again, the simulation results are as expected. Both schools performed poorly but the school with the lowest causal impact performed the worst. Next however, we introduce selectivity to the same schools and students.</p>
<pre><code>relative_2 = {
schools: [{impact: -0.25}, {impact: -0.5}],
selectivity: 0.5
}</code></pre>
<figure class="simulation">
<div id="relative-2">
</div>
<figcaption>
Simulation 5: Negative impact with selectivity.
</figcaption>
</figure>
<p>When selectivity is present, the inference from student achievement and school performance breaks down again. The school with the negative but better causal impact performs far better than its causal impact says it should - increasing average student achievement from <code>0.4</code> in Simulation 4 to <code>0.5</code> in Simulation 5. Meanwhile, the school with the worst causal impact performs much worse than it should - decreasing average student achievement from <code>0.33</code> to <code>0.22</code>. The schools remained the same in both simulations. The only change in the simulations, and therefore only possible cause of the change, was the introduction of selectivity.</p>
<p>As selectivity increases, so too does the disconnect between perceived performance and causal impact. The mere presence of a school with lower causal impact on student achievement combined with selectivity results in the higher achievement students enrolling in the <em>least worst</em> school, thereby <em>inflating</em> that school’s apparent performance. Thus, school performance as measured by student achievement is determined not only by the causal impact of a school but also by the relative performance of other schools. This is not to say that school performance <em>is</em> relative, but rather that the performance of one school is affected by the performance of another, whenever selectivity is present. Again, the inference from student achievement to school performance is unreliable.</p>
<h2 id="initial-conditions-matter">Initial Conditions Matter</h2>
<p>In an egalitarian utopia, every child would have the same opportunities as every other. The biological lottery wouldn’t affect educational outcomes. Obviously, this is not our reality — initial conditions matter. Until now, we have looked at scenarios where both schools started with similar initial random distributions of students. In the next simulations, we add a new parameter, <code>skew</code>, that alters the initial enrolment of students.</p>
<p>We continue with the same 1000 students of random achievement (mean <code>0.5</code>) but set the initial distribution between schools. A <code>skew</code> value of <code>0.75</code> here means that 75% of the above average, and therefore only 25% of the below average students, will initially be enrolled in the first school. The default value of <code>0.5</code> means there is an equal distribution of achievement.</p>
<p>In the next simulation, we stipulate that schools have different causal impact but the initial distribution of student achievement is skewed in favour of the school with the lowest impact. No selectivity is present so we should expect the initial mismatch between school impact and performance to be corrected over time.</p>
<pre><code>head_start_1 = {
schools: [{impact: -0.25}, {impact: 0.25}],
selectivity: 0.0,
skew: 0.75
}</code></pre>
<figure class="simulation">
<div id="head-start-1">
</div>
<figcaption>
Simulation 6: Mixed impact with skew.
</figcaption>
</figure>
<p>As expected, any imbalance in the initial distribution when selectivity is absent is eliminated given enough time. In each tick, the causal impact of schools affects student achievement, and every enrolling cohort normalises school performance to a degree. The first school, while appearing to perform strongest initially, eventually performs poorly, with the opposite occurring in the second school.</p>
<p>Next, we add selectivity to the same parameters and see how this changes performance.</p>
<pre><code>head_start_2 = {
schools: [{impact: -0.25}, {impact: 0.25}],
selectivity: 0.5,
skew: 0.75
}</code></pre>
<figure class="simulation">
<div id="head-start-2">
</div>
<figcaption>
Simulation 7: Mixed impact with skew and selection.
</figcaption>
</figure>
<p>Again, when selectivity is present school performance no longer reflects a school’s stipulated causal impact. A skewed initial distribution of students and selectivity results in a school with a negative causal impact performing better than a school with a postive causal impact. Given a sufficiently skewed initial distribution, schools with negative causal impact can continue to outperform schools with positive impact. Yet again, the presence of selectivity in the school system undermines the student achievement to school performance inference mechanism.</p>
<h2 id="discussion">Discussion</h2>
<p>Measuring school performance is important but we cannot measure it directly. Instead, we typically rely on student achievement as a proxy, and <em>infer</em> school performance. This of course, raises a number of epistemic challenges. Few people if any would claim that current school performance measurement regimes that rely on student achievement as proxy data from standardised tests such as NAPLAN are perfect. What these simulations demonstrate however, is that <em>if selectivity is present, then the inference from student achievement to school performance is unwarranted</em>. If a causal inference mechanism isn’t warranted when causes are known with certainty, then it seems highly impausible that the inference can be warranted when causes aren’t known or observable.</p>
<p>The inference from student achievement to school performance is poor even under the very best epistemic conditions. The model above allows us to stipulate the behaviour of much that is uncontrollable in the real world. Within the model, our knowledge of student achievement is perfect and not affected by test errors, strategic teaching, or exam stress. So too is our knowledge of the causal mechanism. As a non-ecological model, we have stipulated that the only way student achievement can change in the simulation is via school impact. We could not ask for more perfect conditions from which to <em>infer</em> causal impact yet even here the link between proxy evidence and actual cause breaks down. <em>We should therefore be very skeptical about inferring school performance from student achievement</em>.</p>
<p>This is of course, a purely theoretical argument. It remains an open empiric question just how well the simulation models reality, and how much choice or selectivity occurs within our school systems. Social scientists are likely to be better placed answering this question than philosophers are, but a few observations about the Australian context are germane. Approximately 35% of all students currently attend a non-government school <span class="citation" data-cites="abs2014">(Australian Bureau of Statistics, <a href="#ref-abs2014">2014</a>)</span> indicating a minimum level of explicit selectivity. Then there is selectivity within government schools. Data on what percentage of students attend their <em>nearest</em> school is difficult to find, as is data on parents moving to different school catchment zones because of perceived school performance, but anecdotal evidence suggests this too occurs.</p>
<p>One might object to my conclusion by claiming that the model described doesn’t accurately reflect the real world. This is certainly true but not grounds to reject the conclusion. All models are abstractions and simplifications of reality - that is their strength. In this case, the model described presents a best case epistemic scenario for measuring school performance. As such, the simplification of the model <em>strengthens</em> the claims for epistemic skepticism towards school performance.</p>
<p>Despite its simplicity, the model also appears to conform with empiric observation. In the sanity check simulations, the model performed exactly as expected. When selectivity was introduced, the simulation was compatible with recent Australian data. <span class="citation" data-cites="nghiem2015">Nghiem et al. (<a href="#ref-nghiem2015">2015</a>)</span> analysis of the Longitudinal Study of Australian Children showed that non-government (and therefore selectively enrolled) schools had higher average student NAPLAN results but once controlled, performed equally with non-government schools or worse in the case of Catholic ones.</p>
<p>Some might also simply dispute my claim that student achievement is typically used for measuring school performance. The NAPLAN, Peabody, and Wechsler tests for exapmle, are intended for measuring student progress, not school performance. Longitudinal <span class="citation" data-cites="zubrick2000">(Zubrick et al., <a href="#ref-zubrick2000">2000</a>)</span>, predictive <span class="citation" data-cites="lavin1965">(Lavin, <a href="#ref-lavin1965">1965</a>)</span>, and psychological studies of academic performance <span class="citation" data-cites="pintrich1990">(Pintrich & De Groot, <a href="#ref-pintrich1990">1990</a>)</span> for example, all rely on these measures without making claims about school performance. In short, one might argue that while my conclusion may be entailed, it is poorly targeted. Yet while it is certainly true that student achievement can be used for much more than assessing school performance, student achievement is still used to infer school performance by parents, teachers, and policy makers alike. NAPLAN as just one example is explicit on this - “[s]chools can gain detailed information about how they are performing, and they can identify strengths and weaknesses which may warrant further attention.” <span class="citation" data-cites="naplan2015">(NAPLAN, <a href="#ref-naplan2015">2015</a>)</span></p>
<p>The impact of student intake and composition has long been acknowledged to play an important role in determining school performance <span class="citation" data-cites="thrupp1995">(Thrupp, <a href="#ref-thrupp1995">1995</a>, <span class="citation" data-cites="thrupp1999">Thrupp (<a href="#ref-thrupp1999">1999</a>)</span>, <span class="citation" data-cites="thrupp2006">Thrupp & Lupton (<a href="#ref-thrupp2006">2006</a>)</span>)</span>. What this simulation so graphically demonstrates is how these common scenarios further complicate research into school performance by undermining our inference mechanisms even in ideal settings.</p>
<p>In short, we should be very skeptical about inferring school performance from student achievement, especially when selectivity is present.</p>
<hr />
<h1 id="appendix">Appendix</h1>
<p>This article is written in Literate Coffeescript. Literate programming has a great deal to offer the humanities, not least of which is that it makes replication available to all readers. To build the simulation from the raw paper, download the project from <a href="https://github.com/blind-review/school-performance">the repo</a> and type <code>make paper</code> in the command line. Make sure you’ve got <a href="http://Coffeescript.org/">Coffeescript</a> and <a href="http://pandoc.org">Pandoc</a> installed first.</p>
<p>Running a simulation with your own parameters is easy. You just need to specify the parameters and call <code>display(target-location, parameters)</code> and add a target location in HTML <code>id="target-location"</code></p>
<h2 id="browser-code">Browser Code</h2>
<p>Much of the browser code required in this simulation is not germane to the argument and so has been extracted here to the appendix. It provided here so that the entire simulation is self contained within the paper itself.</p>
<p>To make things look pretty, we will use the <a href="http://d3js.org/">D3.js data visualisation library</a> by Mike Bostock. We will also set some global variables from the browser such as height and width.</p>
<pre><code>d3 = require 'd3'
width = window.innerWidth || 600
height = width * 0.4</code></pre>
<p>Because we will be running multiple simulations in the browser, we will need a way of creating different ones. Here we define a <code>display</code> method for creating a simulation and binding it to a canvas with click events. When a simulation canvas is clicked, the interval runner starts and calls the <code>tick</code> method every 1000 milliseconds.</p>
<pre><code>display = (id, params) =>
runner = false
simulation = new Simulation params
canvas = d3.select("##{id}")
.append("svg:svg")
.attr "height", "auto"
.attr "width", width
.attr "preserveAspectRatio", "xMidYMid meet"
.attr "viewBox", "0 0 #{width} #{Math.max(width * 0.4, height * 0.8)}"
.on "click", () ->
if runner
clearInterval runner
runner = false
else
runner = setInterval () ->
tick simulation
, 1000
canvas.append('text')
.attr "y", () -> height * .85
.attr "x", () -> width * .33</code></pre>
<p>In every tick cycle, we run call simulation’s <code>teach</code> and <code>graduate</code> methods. We then <code>render</code> the simulation to calculate the x & y coordinates for the students, and update the canvas.</p>
<pre><code> tick = (simulation) ->
simulation.tick()
render simulation
draw canvas.selectAll "circle" </code></pre>
<p>We then <code>draw</code> the students on the canvas. We will represent our students as coloured circles and schools by student proximity. We will also append the school averages in text form.</p>
<pre><code> draw = (students) ->
students.transition()
.duration 1000
.style "fill", (d) -> colour d, 'achievement'
.style "opacity", 0.5
.attr "r", 8
.attr "cx", (d) -> d.x
.attr "cy", (d) -> d.y
canvas.select "text"
.text "#{simulation.schools[0].score.toFixed(5)}
- Average Student achievement
- #{simulation.schools[1].score.toFixed(5)}" </code></pre>
<p>Finally, we setup the initial rendering.</p>
<pre><code> setup = () ->
render simulation
students = canvas.selectAll "circle"
.data simulation.students
students.enter()
.append "circle"
.style "fill", (student) ->
colour student, 'achievement'
.style "opacity", 0.5
.attr "r", 8
.attr "cx", (d) -> d.x
.attr "cy", (d) -> d.y
setup()</code></pre>
<p>In the browser code above, we have relied on a few helper methods. The first of these represents student achievement graphically using colour. Blue represents high achievement and red low achievement. With a little bit of maths, we can convert achievement on a range of 0.0 to 1.0 to a hexadecimal representation of Red-Green-Blue colour.</p>
<pre><code>colour = (d, attribute) ->
red = Math.floor( (1-d[attribute])*255 ).toString 16
blue = Math.floor( d[attribute]*255 ).toString 16
red = "0#{red}" if red.length is 1
blue = "0#{blue}" if blue.length is 1
"##{red}00#{blue}"</code></pre>
<p>To display students and indicate school enrolment by student proximity, we create a Gaussian overlay so all students in the same school clump together. Each student is assigned a randomised position centred on their school.</p>
<pre><code>render = (simulation) ->
simulation.schools.map (school) ->
school.x = width * (0.3 + 0.5 * school.id)
school.y = height * 0.6
simulation.students.map (student) ->
student.x = gausian(width/1.2) + student.school.x
student.y = gausian(height/0.5) + student.school.y
gausian = (range) ->
Math.random()*range/8 + Math.random()*range/8 + Math.random()*range/8 - range/4</code></pre>
<p>Finally, we trigger the various simulations outlined in the article once all the scripts have loaded.</p>
<pre><code>window.onload = () ->
display 'sanity-check-1', sanity_check_1
display 'sanity-check-2', sanity_check_2
display 'shifting-averages', shifting_averages
display 'relative-1', relative_1
display 'relative-2', relative_2
display 'head-start-1', head_start_1
display 'head-start-2', head_start_2</code></pre>
<h2 id="bibliography" class="unnumbered">Bibliography</h2>
<div id="refs" class="references">
<div id="ref-adolphus2013">
<p>Adolphus, K., Lawton, C. L., & Dye, L. (2013). The effects of breakfast on behavior and academic performance in children and adolescents. <em>Frontiers in Human Neuroscience</em>, <em>7</em>.</p>
</div>
<div id="ref-abs2014">
<p>Australian Bureau of Statistics. (2014). <em>4221.0 - schools, australia, 2014 dataset</em>. Retrieved from <a href="http://www.abs.gov.au/AUSSTATS/[email protected]/Lookup/4221.0Main+Features12014?OpenDocument" class="uri">http://www.abs.gov.au/AUSSTATS/[email protected]/Lookup/4221.0Main+Features12014?OpenDocument</a></p>
</div>
<div id="ref-bronfenbrenner1994">
<p>Bronfenbrenner, U. (1994). Ecological models of human development. <em>Readings on the Development of Children</em>, <em>2</em>, 37–43.</p>
</div>
<div id="ref-caro2009">
<p>Caro, D. H., McDonald, J. T., & Willms, J. D. (2009). Socio-economic status and academic achievement trajectories from childhood to adolescence. <em>Canadian Journal of Education/Revue Canadienne de L’éducation</em>, <em>32</em>(3), 558–590.</p>
</div>
<div id="ref-dunn1965">
<p>Dunn, L. M., Dunn, L. M., Bulheller, S., & Häcker, H. (1965). <em>Peabody picture vocabulary test</em>. American Guidance Service Circle Pines, MN.</p>
</div>
<div id="ref-ehrenberg2001">
<p>Ehrenberg, R. G., Brewer, D. J., Gamoran, A., & Willms, J. D. (2001). Class size and student achievement. <em>Psychological Science in the Public Interest</em>, 1–30.</p>
</div>
<div id="ref-hanushek2003">
<p>Hanushek, E. A., Kain, J. F., Markman, J. M., & Rivkin, S. G. (2003). Does peer ability affect student achievement? <em>Journal of Applied Econometrics</em>, <em>18</em>(5), 527–544.</p>
</div>
<div id="ref-hitchcock2019">
<p>Hitchcock, C. (2019). Causal models. In E. N. Zalta (Ed.), <em>The stanford encyclopedia of philosophy</em> (Summer 2019). Retrieved from <a href="https://plato.stanford.edu/archives/sum2019/entries/causal-models/" class="uri">https://plato.stanford.edu/archives/sum2019/entries/causal-models/</a></p>
</div>
<div id="ref-hume1748">
<p>Hume, D. (1748). <em>An enquiry concerning human understanding</em>.</p>
</div>
<div id="ref-kahneman2011">
<p>Kahneman, D. (2011). <em>Thinking, fast and slow</em>. Macmillan.</p>
</div>
<div id="ref-knuth1984">
<p>Knuth, D. E. (1984). Literate programming. <em>The Computer Journal</em>, <em>27</em>(2), 97–111.</p>
</div>
<div id="ref-kosgei2013">
<p>Kosgei, A., Mise, J. K., Odera, O., & Ayugi, M. E. (2013). Influence of teacher characteristics on students’ academic achievement among secondary schools. <em>Journal of Education and Practice</em>, <em>4</em>(3), 76–82.</p>
</div>
<div id="ref-lavin1965">
<p>Lavin, D. E. (1965). <em>The prediction of academic performance.</em></p>
</div>
<div id="ref-naplan2015">
<p>NAPLAN. (2015, October). Why nap. Retrieved from <a href="http://www.nap.edu.au/about/why-nap.html" class="uri">http://www.nap.edu.au/about/why-nap.html</a></p>
</div>
<div id="ref-nghiem2015">
<p>Nghiem, H. S., Nguyen, H. T., Khanam, R., & Connelly, L. B. (2015). Does school type affect cognitive and non-cognitive development in children? Evidence from australian primary schools. <em>Labour Economics</em>, <em>33</em>, 55–65.</p>
</div>
<div id="ref-pintrich1990">
<p>Pintrich, P. R., & De Groot, E. V. (1990). Motivational and self-regulated learning components of classroom academic performance. <em>Journal of Educational Psychology</em>, <em>82</em>(1), 33.</p>
</div>
<div id="ref-schaffer2016">
<p>Schaffer, J. (2016). The metaphysics of causation. In E. N. Zalta (Ed.), <em>The stanford encyclopedia of philosophy</em> (Fall 2016). Retrieved from <a href="https://plato.stanford.edu/archives/fall2016/entries/causation-metaphysics/" class="uri">https://plato.stanford.edu/archives/fall2016/entries/causation-metaphysics/</a></p>
</div>
<div id="ref-thrupp1995">
<p>Thrupp, M. (1995). The school mix effect: The history of an enduring problem in educational research, policy and practice. <em>British Journal of Sociology of Education</em>, <em>16</em>(2), 183–203.</p>
</div>
<div id="ref-thrupp1999">
<p>Thrupp, M. (1999). <em>Schools making a difference: School mix, school effectiveness, and the social limits of reform</em>. McGraw-Hill Education (UK).</p>
</div>
<div id="ref-thrupp2006">
<p>Thrupp, M., & Lupton, R. (2006). TAKING school contexts more seriously: THE social justice challenge. <em>British Journal of Educational Studies</em>, <em>54</em>(3), 308–328. <a href="https://doi.org/10.1111/j.1467-8527.2006.00348.x" class="uri">https://doi.org/10.1111/j.1467-8527.2006.00348.x</a></p>
</div>
<div id="ref-wechsler2003">
<p>Wechsler, D. (2003). Wechsler intelligence scale for children–Fourth edition (wisc-iv). <em>San Antonio, TX: The Psychological Corporation</em>.</p>
</div>
<div id="ref-zubrick2000">
<p>Zubrick, S., Silburn, S., Williams, A., & Vimpani, G. (2000). <em>Indicators of social and family functioning: Final report</em>. Commonwealth Department of Family and Community Services, Canberra. Available at http://www. facs. gov. au/internet/facsin ternt. nsfl aboutfac si programs/families/isff. htm,[2000, 6 Sept.].</p>
</div>
</div>
<section class="footnotes">
<hr />
<ol>
<li id="fn1"><p>Again, the claim is not that schooling is the only causal factor in student performance in the <em>real world</em>, but that it is the only causal factor in the model because it has been programmed as such.<a href="#fnref1">↩</a></p></li>
<li id="fn2"><p>This is not to say that persuasion should trump logical entailment but that a persuasive and valid argument is better than a valid one alone. <span class="citation" data-cites="kahneman2011">Kahneman (<a href="#ref-kahneman2011">2011</a>)</span> (p9) for example attributes the broad appeal of his and Tversky’s work on heuristics biases to the inclusion of demonstrations that the reader could experience themself.<a href="#fnref2">↩</a></p></li>
</ol>
</section>
<script>
(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
!function() {
var d3 = {
version: "3.5.17"
};
var d3_arraySlice = [].slice, d3_array = function(list) {
return d3_arraySlice.call(list);
};
var d3_document = this.document;
function d3_documentElement(node) {
return node && (node.ownerDocument || node.document || node).documentElement;
}
function d3_window(node) {
return node && (node.ownerDocument && node.ownerDocument.defaultView || node.document && node || node.defaultView);
}
if (d3_document) {
try {
d3_array(d3_document.documentElement.childNodes)[0].nodeType;
} catch (e) {
d3_array = function(list) {
var i = list.length, array = new Array(i);
while (i--) array[i] = list[i];
return array;
};
}
}
if (!Date.now) Date.now = function() {
return +new Date();
};
if (d3_document) {
try {
d3_document.createElement("DIV").style.setProperty("opacity", 0, "");
} catch (error) {
var d3_element_prototype = this.Element.prototype, d3_element_setAttribute = d3_element_prototype.setAttribute, d3_element_setAttributeNS = d3_element_prototype.setAttributeNS, d3_style_prototype = this.CSSStyleDeclaration.prototype, d3_style_setProperty = d3_style_prototype.setProperty;
d3_element_prototype.setAttribute = function(name, value) {
d3_element_setAttribute.call(this, name, value + "");
};
d3_element_prototype.setAttributeNS = function(space, local, value) {
d3_element_setAttributeNS.call(this, space, local, value + "");
};
d3_style_prototype.setProperty = function(name, value, priority) {
d3_style_setProperty.call(this, name, value + "", priority);
};
}
}
d3.ascending = d3_ascending;
function d3_ascending(a, b) {
return a < b ? -1 : a > b ? 1 : a >= b ? 0 : NaN;
}
d3.descending = function(a, b) {
return b < a ? -1 : b > a ? 1 : b >= a ? 0 : NaN;
};
d3.min = function(array, f) {
var i = -1, n = array.length, a, b;
if (arguments.length === 1) {
while (++i < n) if ((b = array[i]) != null && b >= b) {
a = b;
break;
}
while (++i < n) if ((b = array[i]) != null && a > b) a = b;
} else {
while (++i < n) if ((b = f.call(array, array[i], i)) != null && b >= b) {
a = b;
break;
}
while (++i < n) if ((b = f.call(array, array[i], i)) != null && a > b) a = b;
}
return a;
};
d3.max = function(array, f) {
var i = -1, n = array.length, a, b;
if (arguments.length === 1) {
while (++i < n) if ((b = array[i]) != null && b >= b) {
a = b;
break;
}
while (++i < n) if ((b = array[i]) != null && b > a) a = b;
} else {
while (++i < n) if ((b = f.call(array, array[i], i)) != null && b >= b) {
a = b;
break;
}
while (++i < n) if ((b = f.call(array, array[i], i)) != null && b > a) a = b;
}
return a;
};
d3.extent = function(array, f) {
var i = -1, n = array.length, a, b, c;
if (arguments.length === 1) {
while (++i < n) if ((b = array[i]) != null && b >= b) {
a = c = b;
break;
}
while (++i < n) if ((b = array[i]) != null) {
if (a > b) a = b;
if (c < b) c = b;
}
} else {
while (++i < n) if ((b = f.call(array, array[i], i)) != null && b >= b) {
a = c = b;
break;
}
while (++i < n) if ((b = f.call(array, array[i], i)) != null) {
if (a > b) a = b;
if (c < b) c = b;
}
}
return [ a, c ];
};
function d3_number(x) {
return x === null ? NaN : +x;
}
function d3_numeric(x) {
return !isNaN(x);
}
d3.sum = function(array, f) {
var s = 0, n = array.length, a, i = -1;
if (arguments.length === 1) {
while (++i < n) if (d3_numeric(a = +array[i])) s += a;
} else {
while (++i < n) if (d3_numeric(a = +f.call(array, array[i], i))) s += a;
}
return s;
};
d3.mean = function(array, f) {
var s = 0, n = array.length, a, i = -1, j = n;
if (arguments.length === 1) {
while (++i < n) if (d3_numeric(a = d3_number(array[i]))) s += a; else --j;
} else {
while (++i < n) if (d3_numeric(a = d3_number(f.call(array, array[i], i)))) s += a; else --j;
}
if (j) return s / j;
};
d3.quantile = function(values, p) {
var H = (values.length - 1) * p + 1, h = Math.floor(H), v = +values[h - 1], e = H - h;
return e ? v + e * (values[h] - v) : v;
};
d3.median = function(array, f) {
var numbers = [], n = array.length, a, i = -1;
if (arguments.length === 1) {
while (++i < n) if (d3_numeric(a = d3_number(array[i]))) numbers.push(a);
} else {
while (++i < n) if (d3_numeric(a = d3_number(f.call(array, array[i], i)))) numbers.push(a);
}
if (numbers.length) return d3.quantile(numbers.sort(d3_ascending), .5);
};
d3.variance = function(array, f) {
var n = array.length, m = 0, a, d, s = 0, i = -1, j = 0;
if (arguments.length === 1) {
while (++i < n) {
if (d3_numeric(a = d3_number(array[i]))) {
d = a - m;
m += d / ++j;
s += d * (a - m);
}
}
} else {
while (++i < n) {
if (d3_numeric(a = d3_number(f.call(array, array[i], i)))) {
d = a - m;
m += d / ++j;
s += d * (a - m);
}
}
}
if (j > 1) return s / (j - 1);
};
d3.deviation = function() {
var v = d3.variance.apply(this, arguments);
return v ? Math.sqrt(v) : v;
};
function d3_bisector(compare) {
return {
left: function(a, x, lo, hi) {
if (arguments.length < 3) lo = 0;
if (arguments.length < 4) hi = a.length;
while (lo < hi) {
var mid = lo + hi >>> 1;
if (compare(a[mid], x) < 0) lo = mid + 1; else hi = mid;
}
return lo;
},
right: function(a, x, lo, hi) {
if (arguments.length < 3) lo = 0;
if (arguments.length < 4) hi = a.length;
while (lo < hi) {
var mid = lo + hi >>> 1;
if (compare(a[mid], x) > 0) hi = mid; else lo = mid + 1;
}
return lo;
}
};
}
var d3_bisect = d3_bisector(d3_ascending);
d3.bisectLeft = d3_bisect.left;
d3.bisect = d3.bisectRight = d3_bisect.right;
d3.bisector = function(f) {
return d3_bisector(f.length === 1 ? function(d, x) {
return d3_ascending(f(d), x);
} : f);
};
d3.shuffle = function(array, i0, i1) {
if ((m = arguments.length) < 3) {
i1 = array.length;
if (m < 2) i0 = 0;
}
var m = i1 - i0, t, i;
while (m) {
i = Math.random() * m-- | 0;
t = array[m + i0], array[m + i0] = array[i + i0], array[i + i0] = t;
}
return array;
};
d3.permute = function(array, indexes) {
var i = indexes.length, permutes = new Array(i);
while (i--) permutes[i] = array[indexes[i]];
return permutes;
};
d3.pairs = function(array) {
var i = 0, n = array.length - 1, p0, p1 = array[0], pairs = new Array(n < 0 ? 0 : n);
while (i < n) pairs[i] = [ p0 = p1, p1 = array[++i] ];
return pairs;
};
d3.transpose = function(matrix) {
if (!(n = matrix.length)) return [];
for (var i = -1, m = d3.min(matrix, d3_transposeLength), transpose = new Array(m); ++i < m; ) {
for (var j = -1, n, row = transpose[i] = new Array(n); ++j < n; ) {
row[j] = matrix[j][i];
}
}
return transpose;
};
function d3_transposeLength(d) {
return d.length;
}
d3.zip = function() {
return d3.transpose(arguments);
};
d3.keys = function(map) {
var keys = [];
for (var key in map) keys.push(key);
return keys;
};
d3.values = function(map) {
var values = [];
for (var key in map) values.push(map[key]);
return values;
};
d3.entries = function(map) {
var entries = [];
for (var key in map) entries.push({
key: key,
value: map[key]
});
return entries;
};
d3.merge = function(arrays) {
var n = arrays.length, m, i = -1, j = 0, merged, array;
while (++i < n) j += arrays[i].length;
merged = new Array(j);
while (--n >= 0) {
array = arrays[n];
m = array.length;
while (--m >= 0) {
merged[--j] = array[m];
}
}
return merged;
};
var abs = Math.abs;
d3.range = function(start, stop, step) {
if (arguments.length < 3) {
step = 1;
if (arguments.length < 2) {
stop = start;
start = 0;
}
}
if ((stop - start) / step === Infinity) throw new Error("infinite range");
var range = [], k = d3_range_integerScale(abs(step)), i = -1, j;
start *= k, stop *= k, step *= k;
if (step < 0) while ((j = start + step * ++i) > stop) range.push(j / k); else while ((j = start + step * ++i) < stop) range.push(j / k);
return range;
};
function d3_range_integerScale(x) {
var k = 1;
while (x * k % 1) k *= 10;
return k;
}
function d3_class(ctor, properties) {
for (var key in properties) {
Object.defineProperty(ctor.prototype, key, {
value: properties[key],
enumerable: false
});
}
}
d3.map = function(object, f) {
var map = new d3_Map();
if (object instanceof d3_Map) {
object.forEach(function(key, value) {
map.set(key, value);
});
} else if (Array.isArray(object)) {
var i = -1, n = object.length, o;
if (arguments.length === 1) while (++i < n) map.set(i, object[i]); else while (++i < n) map.set(f.call(object, o = object[i], i), o);
} else {
for (var key in object) map.set(key, object[key]);
}
return map;
};
function d3_Map() {
this._ = Object.create(null);
}
var d3_map_proto = "__proto__", d3_map_zero = "\x00";
d3_class(d3_Map, {
has: d3_map_has,
get: function(key) {
return this._[d3_map_escape(key)];
},
set: function(key, value) {
return this._[d3_map_escape(key)] = value;
},
remove: d3_map_remove,
keys: d3_map_keys,
values: function() {
var values = [];
for (var key in this._) values.push(this._[key]);
return values;
},
entries: function() {
var entries = [];
for (var key in this._) entries.push({
key: d3_map_unescape(key),
value: this._[key]
});
return entries;
},
size: d3_map_size,
empty: d3_map_empty,
forEach: function(f) {
for (var key in this._) f.call(this, d3_map_unescape(key), this._[key]);
}
});
function d3_map_escape(key) {
return (key += "") === d3_map_proto || key[0] === d3_map_zero ? d3_map_zero + key : key;
}
function d3_map_unescape(key) {
return (key += "")[0] === d3_map_zero ? key.slice(1) : key;
}
function d3_map_has(key) {
return d3_map_escape(key) in this._;
}
function d3_map_remove(key) {
return (key = d3_map_escape(key)) in this._ && delete this._[key];
}
function d3_map_keys() {
var keys = [];
for (var key in this._) keys.push(d3_map_unescape(key));
return keys;
}
function d3_map_size() {
var size = 0;
for (var key in this._) ++size;
return size;
}
function d3_map_empty() {
for (var key in this._) return false;
return true;
}
d3.nest = function() {
var nest = {}, keys = [], sortKeys = [], sortValues, rollup;
function map(mapType, array, depth) {
if (depth >= keys.length) return rollup ? rollup.call(nest, array) : sortValues ? array.sort(sortValues) : array;
var i = -1, n = array.length, key = keys[depth++], keyValue, object, setter, valuesByKey = new d3_Map(), values;
while (++i < n) {
if (values = valuesByKey.get(keyValue = key(object = array[i]))) {
values.push(object);
} else {
valuesByKey.set(keyValue, [ object ]);
}
}
if (mapType) {
object = mapType();
setter = function(keyValue, values) {
object.set(keyValue, map(mapType, values, depth));
};
} else {
object = {};
setter = function(keyValue, values) {
object[keyValue] = map(mapType, values, depth);
};
}
valuesByKey.forEach(setter);
return object;
}
function entries(map, depth) {
if (depth >= keys.length) return map;
var array = [], sortKey = sortKeys[depth++];
map.forEach(function(key, keyMap) {
array.push({
key: key,
values: entries(keyMap, depth)
});
});
return sortKey ? array.sort(function(a, b) {
return sortKey(a.key, b.key);
}) : array;
}
nest.map = function(array, mapType) {
return map(mapType, array, 0);
};
nest.entries = function(array) {
return entries(map(d3.map, array, 0), 0);
};
nest.key = function(d) {
keys.push(d);
return nest;
};
nest.sortKeys = function(order) {
sortKeys[keys.length - 1] = order;
return nest;
};
nest.sortValues = function(order) {
sortValues = order;
return nest;
};
nest.rollup = function(f) {
rollup = f;
return nest;
};
return nest;
};
d3.set = function(array) {
var set = new d3_Set();
if (array) for (var i = 0, n = array.length; i < n; ++i) set.add(array[i]);
return set;
};
function d3_Set() {
this._ = Object.create(null);
}
d3_class(d3_Set, {
has: d3_map_has,
add: function(key) {
this._[d3_map_escape(key += "")] = true;
return key;
},
remove: d3_map_remove,
values: d3_map_keys,
size: d3_map_size,
empty: d3_map_empty,
forEach: function(f) {
for (var key in this._) f.call(this, d3_map_unescape(key));
}
});
d3.behavior = {};
function d3_identity(d) {
return d;
}
d3.rebind = function(target, source) {
var i = 1, n = arguments.length, method;
while (++i < n) target[method = arguments[i]] = d3_rebind(target, source, source[method]);
return target;
};
function d3_rebind(target, source, method) {