%!PS-Adobe-2.0
%%Creator: dvips 5.83 (MiKTeX 1.20) Copyright 1998 Radical Eye Software
%%Title: subjective-errors.dvi
%%CreationDate: Sun May 02 04:54:06 1999
%%Pages: 12
%%PageOrder: Ascend
%%BoundingBox: 0 0 596 842
%%DocumentFonts: Times-Bold Times-Roman Times-Italic Helvetica-Bold
%%EndComments
%DVIPSWebPage: (www.radicaleye.com)
%DVIPSCommandLine: dvips subjective-errors.dvi -o subjective-errors.ps
%DVIPSParameters: dpi=600, compressed
%DVIPSSource:  TeX output 1999.05.02:0453
%%BeginProcSet: texc.pro
%!
/TeXDict 300 dict def TeXDict begin/N{def}def/B{bind def}N/S{exch}N/X{S
N}B/A{dup}B/TR{translate}N/isls false N/vsize 11 72 mul N/hsize 8.5 72
mul N/landplus90{false}def/@rigin{isls{[0 landplus90{1 -1}{-1 1}ifelse 0
0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{
landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize
mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR[
matrix currentmatrix{A A round sub abs 0.00001 lt{round}if}forall round
exch round exch]setmatrix}N/@landscape{/isls true N}B/@manualfeed{
statusdict/manualfeed true put}B/@copies{/#copies X}B/FMat[1 0 0 -1 0 0]
N/FBB[0 0 0 0]N/nn 0 N/IEn 0 N/ctr 0 N/df-tail{/nn 8 dict N nn begin
/FontType 3 N/FontMatrix fntrx N/FontBBox FBB N string/base X array
/BitMaps X/BuildChar{CharBuilder}N/Encoding IEn N end A{/foo setfont}2
array copy cvx N load 0 nn put/ctr 0 N[}B/sf 0 N/df{/sf 1 N/fntrx FMat N
df-tail}B/dfs{div/sf X/fntrx[sf 0 0 sf neg 0 0]N df-tail}B/E{pop nn A
definefont setfont}B/Cw{Cd A length 5 sub get}B/Ch{Cd A length 4 sub get
}B/Cx{128 Cd A length 3 sub get sub}B/Cy{Cd A length 2 sub get 127 sub}
B/Cdx{Cd A length 1 sub get}B/Ci{Cd A type/stringtype ne{ctr get/ctr ctr
1 add N}if}B/id 0 N/rw 0 N/rc 0 N/gp 0 N/cp 0 N/G 0 N/CharBuilder{save 3
1 roll S A/base get 2 index get S/BitMaps get S get/Cd X pop/ctr 0 N Cdx
0 Cx Cy Ch sub Cx Cw add Cy setcachedevice Cw Ch true[1 0 0 -1 -.1 Cx
sub Cy .1 sub]/id Ci N/rw Cw 7 add 8 idiv string N/rc 0 N/gp 0 N/cp 0 N{
rc 0 ne{rc 1 sub/rc X rw}{G}ifelse}imagemask restore}B/G{{id gp get/gp
gp 1 add N A 18 mod S 18 idiv pl S get exec}loop}B/adv{cp add/cp X}B
/chg{rw cp id gp 4 index getinterval putinterval A gp add/gp X adv}B/nd{
/cp 0 N rw exit}B/lsh{rw cp 2 copy get A 0 eq{pop 1}{A 255 eq{pop 254}{
A A add 255 and S 1 and or}ifelse}ifelse put 1 adv}B/rsh{rw cp 2 copy
get A 0 eq{pop 128}{A 255 eq{pop 127}{A 2 idiv S 128 and or}ifelse}
ifelse put 1 adv}B/clr{rw cp 2 index string putinterval adv}B/set{rw cp
fillstr 0 4 index getinterval putinterval adv}B/fillstr 18 string 0 1 17
{2 copy 255 put pop}for N/pl[{adv 1 chg}{adv 1 chg nd}{1 add chg}{1 add
chg nd}{adv lsh}{adv lsh nd}{adv rsh}{adv rsh nd}{1 add adv}{/rc X nd}{
1 add set}{1 add clr}{adv 2 chg}{adv 2 chg nd}{pop nd}]A{bind pop}
forall N/D{/cc X A type/stringtype ne{]}if nn/base get cc ctr put nn
/BitMaps get S ctr S sf 1 ne{A A length 1 sub A 2 index S get sf div put
}if put/ctr ctr 1 add N}B/I{cc 1 add D}B/bop{userdict/bop-hook known{
bop-hook}if/SI save N @rigin 0 0 moveto/V matrix currentmatrix A 1 get A
mul exch 0 get A mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N/eop{
SI restore userdict/eop-hook known{eop-hook}if showpage}N/@start{
userdict/start-hook known{start-hook}if pop/VResolution X/Resolution X
1000 div/DVImag X/IEn 256 array N 2 string 0 1 255{IEn S A 360 add 36 4
index cvrs cvn put}for pop 65781.76 div/vsize X 65781.76 div/hsize X}N
/p{show}N/RMat[1 0 0 -1 0 0]N/BDot 260 string N/Rx 0 N/Ry 0 N/V{}B/RV/v{
/Ry X/Rx X V}B statusdict begin/product where{pop false[(Display)(NeXT)
(LaserWriter 16/600)]{A length product length le{A length product exch 0
exch getinterval eq{pop true exit}if}{pop}ifelse}forall}{false}ifelse
end{{gsave TR -.1 .1 TR 1 1 scale Rx Ry false RMat{BDot}imagemask
grestore}}{{gsave TR -.1 .1 TR Rx Ry scale 1 1 false RMat{BDot}
imagemask grestore}}ifelse B/QV{gsave newpath transform round exch round
exch itransform moveto Rx 0 rlineto 0 Ry neg rlineto Rx neg 0 rlineto
fill grestore}B/a{moveto}B/delta 0 N/tail{A/delta X 0 rmoveto}B/M{S p
delta add tail}B/b{S p tail}B/c{-4 M}B/d{-3 M}B/e{-2 M}B/f{-1 M}B/g{0 M}
B/h{1 M}B/i{2 M}B/j{3 M}B/k{4 M}B/w{0 rmoveto}B/l{p -4 w}B/m{p -3 w}B/n{
p -2 w}B/o{p -1 w}B/q{p 1 w}B/r{p 2 w}B/s{p 3 w}B/t{p 4 w}B/x{0 S
rmoveto}B/y{3 2 roll p a}B/bos{/SS save N}B/eos{SS restore}B end

%%EndProcSet
%%BeginProcSet: pstricks.pro
%!
% PostScript prologue for pstricks.tex.
% Created 1994/6/23. Source file was pstricks.doc
% Version 0.93a, 93/03/12.
% For use with Rokicki's dvips.
/tx@Dict 200 dict def tx@Dict begin
/ADict 25 dict def
/CM { matrix currentmatrix } bind def
/SLW /setlinewidth load def
/CLW /currentlinewidth load def
/CP /currentpoint load def
/ED { exch def } bind def
/L /lineto load def
/T /translate load def
/TMatrix { } def
/RAngle { 0 } def
/Atan { /atan load stopped { pop pop 0 } if } def
/Div { dup 0 eq { pop } { div } ifelse } def
/NET { neg exch neg exch T } def
/Pyth { dup mul exch dup mul add sqrt } def
/PtoC { 2 copy cos mul 3 1 roll sin mul } def
/PathLength@ { /z z y y1 sub x x1 sub Pyth add def /y1 y def /x1 x def }
def
/PathLength { flattenpath /z 0 def { /y1 ED /x1 ED /y2 y1 def /x2 x1 def
} { /y ED /x ED PathLength@ } {} { /y y2 def /x x2 def PathLength@ }
/pathforall load stopped { pop pop pop pop } if z } def
/STP { .996264 dup scale } def
/STV { SDict begin normalscale end STP  } def
/DashLine { dup 0 gt { /a .5 def PathLength exch div } { pop /a 1 def
PathLength } ifelse /b ED /x ED /y ED /z y x add def b a .5 sub 2 mul y
mul sub z Div round z mul a .5 sub 2 mul y mul add b exch Div dup y mul
/y ED x mul /x ED x 0 gt y 0 gt and { [ y x ] 1 a sub y mul } { [ 1 0 ]
0 } ifelse setdash stroke } def
/DotLine { /b PathLength def /a ED /z ED /y CLW def /z y z add def a 0 gt
{ /b b a div def } { a 0 eq { /b b y sub def } { a -3 eq { /b b y add
def } if } ifelse } ifelse [ 0 b b z Div round Div dup 0 le { pop 1 } if
] a 0 gt { 0 } { y 2 div a -2 gt { neg } if } ifelse setdash 1
setlinecap stroke } def
/LineFill { gsave abs CLW add /a ED a 0 dtransform round exch round exch
2 copy idtransform exch Atan rotate idtransform pop /a ED .25 .25
itransform translate pathbbox /y2 ED a Div ceiling cvi /x2 ED /y1 ED a
Div cvi /x1 ED /y2 y2 y1 sub def clip newpath 2 setlinecap systemdict
/setstrokeadjust known { true setstrokeadjust } if x2 x1 sub 1 add { x1
a mul y1 moveto 0 y2 rlineto stroke /x1 x1 1 add def } repeat grestore }
def
/BeginArrow { ADict begin /@mtrx CM def gsave 2 copy T 2 index sub neg
exch 3 index sub exch Atan rotate newpath } def
/EndArrow { @mtrx setmatrix CP grestore end } def
/Arrow { CLW mul add dup 2 div /w ED mul dup /h ED mul /a ED { 0 h T 1 -1
scale } if w neg h moveto 0 0 L w h L w neg a neg rlineto gsave fill
grestore } def
/Tbar { CLW mul add /z ED z -2 div CLW 2 div moveto z 0 rlineto stroke 0
CLW moveto } def
/Bracket { CLW mul add dup CLW sub 2 div /x ED mul CLW add /y ED /z CLW 2
div def x neg y moveto x neg CLW 2 div L x CLW 2 div L x y L stroke 0
CLW moveto } def
/RoundBracket { CLW mul add dup 2 div /x ED mul /y ED /mtrx CM def 0 CLW
2 div T x y mul 0 ne { x y scale } if 1 1 moveto .85 .5 .35 0 0 0
curveto -.35 0 -.85 .5 -1 1 curveto mtrx setmatrix stroke 0 CLW moveto }
def
/SD { 0 360 arc fill } def
/EndDot { { /z DS def } { /z 0 def } ifelse /b ED 0 z DS SD b { 0 z DS
CLW sub SD } if 0 DS z add CLW 4 div sub moveto } def
/Shadow { [ { /moveto load } { /lineto load } { /curveto load } {
/closepath load } /pathforall load stopped { pop pop pop pop CP /moveto
load } if ] cvx newpath 3 1 roll T exec } def
/NArray { aload length 2 div dup dup cvi eq not { exch pop } if /n exch
cvi def } def
/NArray { /f ED counttomark 2 div dup cvi /n ED n eq not { exch pop } if
f { ] aload /Points ED } { n 2 mul 1 add -1 roll pop } ifelse } def
/Line { NArray n 0 eq not { n 1 eq { 0 0 /n 2 def } if ArrowA /n n 2 sub
def n { Lineto } repeat CP 4 2 roll ArrowB L pop pop } if } def
/Arcto { /a [ 6 -2 roll ] cvx def a r /arcto load stopped { 5 } { 4 }
ifelse { pop } repeat a } def
/CheckClosed { dup n 2 mul 1 sub index eq 2 index n 2 mul 1 add index eq
and { pop pop /n n 1 sub def } if } def
/Polygon { NArray n 2 eq { 0 0 /n 3 def } if n 3 lt { n { pop pop }
repeat } { n 3 gt { CheckClosed } if n 2 mul -2 roll /y0 ED /x0 ED /y1
ED /x1 ED x1 y1 /x1 x0 x1 add 2 div def /y1 y0 y1 add 2 div def x1 y1
moveto /n n 2 sub def n { Lineto } repeat x1 y1 x0 y0 6 4 roll Lineto
Lineto pop pop closepath } ifelse } def
/Diamond { /mtrx CM def T rotate /h ED /w ED dup 0 eq { pop } { CLW mul
neg /d ED /a w h Atan def /h d a sin Div h add def /w d a cos Div w add
def } ifelse mark w 2 div h 2 div w 0 0 h neg w neg 0 0 h w 2 div h 2
div /ArrowA { moveto } def /ArrowB { } def false Line closepath mtrx
setmatrix } def
%  Modification Denis GIROU (CNRS/IDRIS - France) <Denis.Girou@idris.fr> - 15 Jan. 1997
% /Triangle { /mtrx CM def translate rotate /h ED 2 div /w ED dup 0 eq {
% pop } { CLW mul /d ED /h h d w h Atan sin Div sub def /w w d h w Atan 2
% div dup cos exch sin Div mul sub def } ifelse mark 0 d w neg d 0 h w d 0
% d /ArrowA { moveto } def /ArrowB { } def false Line closepath mtrx
% setmatrix } def
/Triangle { /mtrx CM def translate rotate /h ED 2 div /w ED dup
CLW mul /d ED /h h d w h Atan sin Div sub def /w w d h w Atan 2
div dup cos exch sin Div mul sub def mark 0 d w neg d 0 h w d 0
d /ArrowA { moveto } def /ArrowB { } def false Line closepath mtrx
setmatrix } def
/CCA { /y ED /x ED 2 copy y sub /dy1 ED x sub /dx1 ED /l1 dx1 dy1 Pyth
def } def
/CCA { /y ED /x ED 2 copy y sub /dy1 ED x sub /dx1 ED /l1 dx1 dy1 Pyth
def } def
/CC { /l0 l1 def /x1 x dx sub def /y1 y dy sub def /dx0 dx1 def /dy0 dy1
def CCA /dx dx0 l1 c exp mul dx1 l0 c exp mul add def /dy dy0 l1 c exp
mul dy1 l0 c exp mul add def /m dx0 dy0 Atan dx1 dy1 Atan sub 2 div cos
abs b exp a mul dx dy Pyth Div 2 div def /x2 x l0 dx mul m mul sub def
/y2 y l0 dy mul m mul sub def /dx l1 dx mul m mul neg def /dy l1 dy mul
m mul neg def } def
/IC { /c c 1 add def c 0 lt { /c 0 def } { c 3 gt { /c 3 def } if }
ifelse /a a 2 mul 3 div 45 cos b exp div def CCA /dx 0 def /dy 0 def }
def
/BOC { IC CC x2 y2 x1 y1 ArrowA CP 4 2 roll x y curveto } def
/NC { CC x1 y1 x2 y2 x y curveto } def
/EOC { x dx sub y dy sub 4 2 roll ArrowB 2 copy curveto } def
/BAC { IC CC x y moveto CC x1 y1 CP ArrowA } def
/NAC { x2 y2 x y curveto CC x1 y1 } def
/EAC { x2 y2 x y ArrowB curveto pop pop } def
/OpenCurve { NArray n 3 lt { n { pop pop } repeat } { BOC /n n 3 sub def
n { NC } repeat EOC } ifelse } def
/AltCurve { { false NArray n 2 mul 2 roll [ n 2 mul 3 sub 1 roll ] aload
/Points ED n 2 mul -2 roll } { false NArray } ifelse n 4 lt { n { pop
pop } repeat } { BAC /n n 4 sub def n { NAC } repeat EAC } ifelse } def
/ClosedCurve { NArray n 3 lt { n { pop pop } repeat } { n 3 gt {
CheckClosed } if 6 copy n 2 mul 6 add 6 roll IC CC x y moveto n { NC }
repeat closepath pop pop } ifelse } def
/SQ { /r ED r r moveto r r neg L r neg r neg L r neg r L fill } def
/ST { /y ED /x ED x y moveto x neg y L 0 x L fill } def
/SP { /r ED gsave 0 r moveto 4 { 72 rotate 0 r L } repeat fill grestore }
def
/FontDot { DS 2 mul dup matrix scale matrix concatmatrix exch matrix
rotate matrix concatmatrix exch findfont exch makefont setfont } def
/Rect { x1 y1 y2 add 2 div moveto x1 y2 lineto x2 y2 lineto x2 y1 lineto
x1 y1 lineto closepath } def
/OvalFrame { x1 x2 eq y1 y2 eq or { pop pop x1 y1 moveto x2 y2 L } { y1
y2 sub abs x1 x2 sub abs 2 copy gt { exch pop } { pop } ifelse 2 div
exch { dup 3 1 roll mul exch } if 2 copy lt { pop } { exch pop } ifelse
/b ED x1 y1 y2 add 2 div moveto x1 y2 x2 y2 b arcto x2 y2 x2 y1 b arcto
x2 y1 x1 y1 b arcto x1 y1 x1 y2 b arcto 16 { pop } repeat closepath }
ifelse } def
/Frame { CLW mul /a ED 3 -1 roll 2 copy gt { exch } if a sub /y2 ED a add
/y1 ED 2 copy gt { exch } if a sub /x2 ED a add /x1 ED 1 index 0 eq {
pop pop Rect } { OvalFrame } ifelse } def
/BezierNArray { /f ED counttomark 2 div dup cvi /n ED n eq not { exch pop
} if n 1 sub neg 3 mod 3 add 3 mod { 0 0 /n n 1 add def } repeat f { ]
aload /Points ED } { n 2 mul 1 add -1 roll pop } ifelse } def
/OpenBezier { BezierNArray n 1 eq { pop pop } { ArrowA n 4 sub 3 idiv { 6
2 roll 4 2 roll curveto } repeat 6 2 roll 4 2 roll ArrowB curveto }
ifelse } def
/ClosedBezier { BezierNArray n 1 eq { pop pop } { moveto n 1 sub 3 idiv {
6 2 roll 4 2 roll curveto } repeat closepath } ifelse } def
/BezierShowPoints { gsave Points aload length 2 div cvi /n ED moveto n 1
sub { lineto } repeat CLW 2 div SLW [ 4 4 ] 0 setdash stroke grestore }
def
/Parab { /y0 exch def /x0 exch def /y1 exch def /x1 exch def /dx x0 x1
sub 3 div def /dy y0 y1 sub 3 div def x0 dx sub y0 dy add x1 y1 ArrowA
x0 dx add y0 dy add x0 2 mul x1 sub y1 ArrowB curveto /Points [ x1 y1 x0
y0 x0 2 mul x1 sub y1 ] def } def
/Grid { newpath /a 4 string def /b ED /c ED /n ED cvi dup 1 lt { pop 1 }
if /s ED s div dup 0 eq { pop 1 } if /dy ED s div dup 0 eq { pop 1 } if
/dx ED dy div round dy mul /y0 ED dx div round dx mul /x0 ED dy div
round cvi /y2 ED dx div round cvi /x2 ED dy div round cvi /y1 ED dx div
round cvi /x1 ED /h y2 y1 sub 0 gt { 1 } { -1 } ifelse def /w x2 x1 sub
0 gt { 1 } { -1 } ifelse def b 0 gt { /z1 b 4 div CLW 2 div add def
/Helvetica findfont b scalefont setfont /b b .95 mul CLW 2 div add def }
if systemdict /setstrokeadjust known { true setstrokeadjust /t { } def }
{ /t { transform 0.25 sub round 0.25 add exch 0.25 sub round 0.25 add
exch itransform } bind def } ifelse gsave n 0 gt { 1 setlinecap [ 0 dy n
div ] dy n div 2 div setdash } { 2 setlinecap } ifelse /i x1 def /f y1
dy mul n 0 gt { dy n div 2 div h mul sub } if def /g y2 dy mul n 0 gt {
dy n div 2 div h mul add } if def x2 x1 sub w mul 1 add dup 1000 gt {
pop 1000 } if { i dx mul dup y0 moveto b 0 gt { gsave c i a cvs dup
stringwidth pop /z2 ED w 0 gt {z1} {z1 z2 add neg} ifelse h 0 gt {b neg}
{z1} ifelse rmoveto show grestore } if dup t f moveto g t L stroke /i i
w add def } repeat grestore gsave n 0 gt { 1 setlinecap [ 0 dx n div ]
dy n div 2 div setdash } { 2 setlinecap } ifelse /i y1 def /f x1 dx mul
n 0 gt { dx n div 2 div w mul sub } if def /g x2 dx mul n 0 gt { dx n
div 2 div w mul add } if def y2 y1 sub h mul 1 add dup 1000 gt { pop
1000 } if { newpath i dy mul dup x0 exch moveto b 0 gt { gsave c i a cvs
dup stringwidth pop /z2 ED w 0 gt {z1 z2 add neg} {z1} ifelse h 0 gt
{z1} {b neg} ifelse rmoveto show grestore } if dup f exch t moveto g
exch t L stroke /i i h add def } repeat grestore } def
/ArcArrow { /d ED /b ED /a ED gsave newpath 0 -1000 moveto clip newpath 0
1 0 0 b grestore c mul /e ED pop pop pop r a e d PtoC y add exch x add
exch r a PtoC y add exch x add exch b pop pop pop pop a e d CLW 8 div c
mul neg d } def
/Ellipse { /mtrx CM def T scale 0 0 1 5 3 roll arc mtrx setmatrix } def
/Rot { CP CP translate 3 -1 roll neg rotate NET  } def
/RotBegin { tx@Dict /TMatrix known not { /TMatrix { } def /RAngle { 0 }
def } if /TMatrix [ TMatrix CM ] cvx def /a ED a Rot /RAngle [ RAngle
dup a add ] cvx def } def
/RotEnd { /TMatrix [ TMatrix setmatrix ] cvx def /RAngle [ RAngle pop ]
cvx def } def
/PutCoor { gsave CP T CM STV exch exec moveto setmatrix CP grestore } def
/PutBegin { /TMatrix [ TMatrix CM ] cvx def CP 4 2 roll T moveto } def
/PutEnd { CP /TMatrix [ TMatrix setmatrix ] cvx def moveto } def
/Uput { /a ED add 2 div /h ED 2 div /w ED /s a sin def /c a cos def /b s
abs c abs 2 copy gt dup /q ED { pop } { exch pop } ifelse def /w1 c b
div w mul def /h1 s b div h mul def q { w1 abs w sub dup c mul abs } {
h1 abs h sub dup s mul abs } ifelse } def
/UUput { /z ED abs /y ED /x ED q { x s div c mul abs y gt } { x c div s
mul abs y gt } ifelse { x x mul y y mul sub z z mul add sqrt z add } { q
{ x s div } { x c div } ifelse abs } ifelse a PtoC h1 add exch w1 add
exch } def
/BeginOL { dup (all) eq exch TheOL eq or { IfVisible not { Visible
/IfVisible true def } if } { IfVisible { Invisible /IfVisible false def
} if } ifelse } def
/InitOL { /OLUnit [ 3000 3000 matrix defaultmatrix dtransform ] cvx def
/Visible { CP OLUnit idtransform T moveto } def /Invisible { CP OLUnit
neg exch neg exch idtransform T moveto } def /BOL { BeginOL } def
/IfVisible true def } def
end

%%EndProcSet
%%BeginProcSet: pst-dots.pro
%!PS-Adobe-2.0
%%Title: Dot Font for PSTricks
%%Creator: Timothy Van Zandt <tvz@Princeton.EDU>
%%Creation Date: May 7, 1993
10 dict dup begin
  /FontType 3 def
  /FontMatrix [ .001 0 0 .001 0 0 ] def
  /FontBBox [ 0 0 0 0 ] def
  /Encoding 256 array def
  0 1 255 { Encoding exch /.notdef put } for
  Encoding
    dup (b) 0 get /Bullet put
    dup (c) 0 get /Circle put
    dup (C) 0 get /BoldCircle put
    dup (u) 0 get /SolidTriangle put
    dup (t) 0 get /Triangle put
    dup (T) 0 get /BoldTriangle put
    dup (r) 0 get /SolidSquare put
    dup (s) 0 get /Square put
    dup (S) 0 get /BoldSquare put
    dup (q) 0 get /SolidPentagon put
    dup (p) 0 get /Pentagon put
    (P) 0 get /BoldPentagon put
  /Metrics 13 dict def
  Metrics begin
    /Bullet 1000 def
    /Circle 1000 def
    /BoldCircle 1000 def
    /SolidTriangle 1344 def
    /Triangle 1344 def
    /BoldTriangle 1344 def
    /SolidSquare 886 def
    /Square 886 def
    /BoldSquare 886 def
    /SolidPentagon 1093.2 def
    /Pentagon 1093.2 def
    /BoldPentagon 1093.2 def
    /.notdef 0 def
  end
  /BBoxes 13 dict def
  BBoxes begin
    /Circle { -550 -550 550 550 } def
    /BoldCircle /Circle load def
    /Bullet /Circle load def
    /Triangle { -571.5 -330 571.5 660 } def
    /BoldTriangle /Triangle load def
    /SolidTriangle /Triangle load def
    /Square { -450 -450 450 450 } def
    /BoldSquare /Square load def
    /SolidSquare /Square load def
    /Pentagon { -546.6 -465 546.6 574.7 } def
    /BoldPentagon /Pentagon load def
    /SolidPentagon /Pentagon load def
    /.notdef { 0 0 0 0 } def
  end
  /CharProcs 20 dict def
  CharProcs begin
    /Adjust {
      2 copy dtransform floor .5 add exch floor .5 add exch idtransform
      3 -1 roll div 3 1 roll exch div exch scale
    } def
    /CirclePath { 0 0 500 0 360 arc closepath } def
    /Bullet { 500 500 Adjust CirclePath fill } def
    /Circle { 500 500 Adjust CirclePath .9 .9 scale CirclePath eofill } def
    /BoldCircle { 500 500 Adjust CirclePath .8 .8 scale CirclePath eofill } def
    /BoldCircle { CirclePath .8 .8 scale CirclePath eofill } def
    /TrianglePath {
      0  660 moveto -571.5 -330 lineto 571.5 -330 lineto closepath
    } def
    /SolidTriangle { TrianglePath fill } def
    /Triangle { TrianglePath .85 .85 scale TrianglePath eofill } def
    /BoldTriangle { TrianglePath .7 .7 scale TrianglePath eofill } def
    /SquarePath {
      -450 450 moveto 450 450 lineto 450 -450 lineto -450 -450 lineto
      closepath
    } def
    /SolidSquare { SquarePath fill } def
    /Square { SquarePath .89 .89 scale SquarePath eofill } def
    /BoldSquare { SquarePath .78 .78 scale SquarePath eofill } def
    /PentagonPath {
      -337.8 -465   moveto
       337.8 -465   lineto
       546.6  177.6 lineto
         0    574.7 lineto
      -546.6  177.6 lineto
      closepath
    } def
    /SolidPentagon { PentagonPath fill } def
    /Pentagon { PentagonPath .89 .89 scale PentagonPath eofill } def
    /BoldPentagon { PentagonPath .78 .78 scale PentagonPath eofill } def
    /.notdef { } def
  end
  /BuildGlyph {
    exch
    begin
      Metrics 1 index get exec 0
      BBoxes 3 index get exec
      setcachedevice
      CharProcs begin load exec end
    end
  } def
  /BuildChar {
    1 index /Encoding get exch get
    1 index /BuildGlyph get exec
  } bind def
end
/PSTricksDotFont exch definefont pop

%%EndProcSet
%%BeginProcSet: 8r.enc
% @@psencodingfile@{
%   author = "S. Rahtz, P. MacKay, Alan Jeffrey, B. Horn, K. Berry",
%   version = "0.6",
%   date = "1 July 1998",
%   filename = "8r.enc",
%   email = "tex-fonts@@tug.org",
%   docstring = "Encoding for TrueType or Type 1 fonts
%                to be used with TeX."
% @}
% 
% Idea is to have all the characters normally included in Type 1 fonts
% available for typesetting. This is effectively the characters in Adobe
% Standard Encoding + ISO Latin 1 + extra characters from Lucida.
% 
% Character code assignments were made as follows:
% 
% (1) the Windows ANSI characters are almost all in their Windows ANSI
% positions, because some Windows users cannot easily reencode the
% fonts, and it makes no difference on other systems. The only Windows
% ANSI characters not available are those that make no sense for
% typesetting -- rubout (127 decimal), nobreakspace (160), softhyphen
% (173). quotesingle and grave are moved just because it's such an
% irritation not having them in TeX positions.
% 
% (2) Remaining characters are assigned arbitrarily to the lower part
% of the range, avoiding 0, 10 and 13 in case we meet dumb software.
% 
% (3) Y&Y Lucida Bright includes some extra text characters; in the
% hopes that other PostScript fonts, perhaps created for public
% consumption, will include them, they are included starting at 0x12.
% 
% (4) Remaining positions left undefined are for use in (hopefully)
% upward-compatible revisions, if someday more characters are generally
% available.
% 
% (5) hyphen appears twice for compatibility with both 
% ASCII and Windows.
% 
/TeXBase1Encoding [
% 0x00 (encoded characters from Adobe Standard not in Windows 3.1)
  /.notdef /dotaccent /fi /fl
  /fraction /hungarumlaut /Lslash /lslash
  /ogonek /ring /.notdef
  /breve /minus /.notdef 
% These are the only two remaining unencoded characters, so may as
% well include them.
  /Zcaron /zcaron 
% 0x10
 /caron /dotlessi 
% (unusual TeX characters available in, e.g., Lucida Bright)
 /dotlessj /ff /ffi /ffl 
 /.notdef /.notdef /.notdef /.notdef
 /.notdef /.notdef /.notdef /.notdef
 % very contentious; it's so painful not having quoteleft and quoteright
 % at 96 and 145 that we move the things normally found there to here.
 /grave /quotesingle 
% 0x20 (ASCII begins)
 /space /exclam /quotedbl /numbersign
 /dollar /percent /ampersand /quoteright
 /parenleft /parenright /asterisk /plus /comma /hyphen /period /slash
% 0x30
 /zero /one /two /three /four /five /six /seven
 /eight /nine /colon /semicolon /less /equal /greater /question
% 0x40
 /at /A /B /C /D /E /F /G /H /I /J /K /L /M /N /O
% 0x50
 /P /Q /R /S /T /U /V /W
 /X /Y /Z /bracketleft /backslash /bracketright /asciicircum /underscore
% 0x60
 /quoteleft /a /b /c /d /e /f /g /h /i /j /k /l /m /n /o
% 0x70
 /p /q /r /s /t /u /v /w
 /x /y /z /braceleft /bar /braceright /asciitilde
 /.notdef % rubout; ASCII ends
% 0x80
 /.notdef /.notdef /quotesinglbase /florin
 /quotedblbase /ellipsis /dagger /daggerdbl
 /circumflex /perthousand /Scaron /guilsinglleft
 /OE /.notdef /.notdef /.notdef
% 0x90
 /.notdef /.notdef /.notdef /quotedblleft
 /quotedblright /bullet /endash /emdash
 /tilde /trademark /scaron /guilsinglright
 /oe /.notdef /.notdef /Ydieresis
% 0xA0
 /.notdef % nobreakspace
 /exclamdown /cent /sterling
 /currency /yen /brokenbar /section
 /dieresis /copyright /ordfeminine /guillemotleft
 /logicalnot
 /hyphen % Y&Y (also at 45); Windows' softhyphen
 /registered
 /macron
% 0xD0
 /degree /plusminus /twosuperior /threesuperior
 /acute /mu /paragraph /periodcentered
 /cedilla /onesuperior /ordmasculine /guillemotright
 /onequarter /onehalf /threequarters /questiondown
% 0xC0
 /Agrave /Aacute /Acircumflex /Atilde /Adieresis /Aring /AE /Ccedilla
 /Egrave /Eacute /Ecircumflex /Edieresis
 /Igrave /Iacute /Icircumflex /Idieresis
% 0xD0
 /Eth /Ntilde /Ograve /Oacute
 /Ocircumflex /Otilde /Odieresis /multiply
 /Oslash /Ugrave /Uacute /Ucircumflex
 /Udieresis /Yacute /Thorn /germandbls
% 0xE0
 /agrave /aacute /acircumflex /atilde
 /adieresis /aring /ae /ccedilla
 /egrave /eacute /ecircumflex /edieresis
 /igrave /iacute /icircumflex /idieresis
% 0xF0
 /eth /ntilde /ograve /oacute
 /ocircumflex /otilde /odieresis /divide
 /oslash /ugrave /uacute /ucircumflex
 /udieresis /yacute /thorn /ydieresis
] def

%%EndProcSet
%%BeginProcSet: texps.pro
%!
TeXDict begin/rf{findfont dup length 1 add dict begin{1 index/FID ne 2
index/UniqueID ne and{def}{pop pop}ifelse}forall[1 index 0 6 -1 roll
exec 0 exch 5 -1 roll VResolution Resolution div mul neg 0 0]/Metrics
exch def dict begin Encoding{exch dup type/integertype ne{pop pop 1 sub
dup 0 le{pop}{[}ifelse}{FontMatrix 0 get div Metrics 0 get div def}
ifelse}forall Metrics/Metrics currentdict end def[2 index currentdict
end definefont 3 -1 roll makefont/setfont cvx]cvx def}def/ObliqueSlant{
dup sin S cos div neg}B/SlantFont{4 index mul add}def/ExtendFont{3 -1
roll mul exch}def/ReEncodeFont{CharStrings rcheck{/Encoding false def
dup[exch{dup CharStrings exch known not{pop/.notdef/Encoding true def}
if}forall Encoding{]exch pop}{cleartomark}ifelse}if/Encoding exch def}
def end

%%EndProcSet
%%BeginProcSet: special.pro
%!
TeXDict begin/SDict 200 dict N SDict begin/@SpecialDefaults{/hs 612 N
/vs 792 N/ho 0 N/vo 0 N/hsc 1 N/vsc 1 N/ang 0 N/CLIP 0 N/rwiSeen false N
/rhiSeen false N/letter{}N/note{}N/a4{}N/legal{}N}B/@scaleunit 100 N
/@hscale{@scaleunit div/hsc X}B/@vscale{@scaleunit div/vsc X}B/@hsize{
/hs X/CLIP 1 N}B/@vsize{/vs X/CLIP 1 N}B/@clip{/CLIP 2 N}B/@hoffset{/ho
X}B/@voffset{/vo X}B/@angle{/ang X}B/@rwi{10 div/rwi X/rwiSeen true N}B
/@rhi{10 div/rhi X/rhiSeen true N}B/@llx{/llx X}B/@lly{/lly X}B/@urx{
/urx X}B/@ury{/ury X}B/magscale true def end/@MacSetUp{userdict/md known
{userdict/md get type/dicttype eq{userdict begin md length 10 add md
maxlength ge{/md md dup length 20 add dict copy def}if end md begin
/letter{}N/note{}N/legal{}N/od{txpose 1 0 mtx defaultmatrix dtransform S
atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{
itransform lineto}}{6 -2 roll transform 6 -2 roll transform 6 -2 roll
transform{itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll
curveto}}{{closepath}}pathforall newpath counttomark array astore/gc xdf
pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}
if}N/txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1
-1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3
get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip
yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub
neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{
noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop
90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get
neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr
1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr
2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4
-1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S
TR}if}N/cp{pop pop showpage pm restore}N end}if}if}N/normalscale{
Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale
}if 0 setgray}N/psfts{S 65781.76 div N}N/startTexFig{/psf$SavedState
save N userdict maxlength dict begin/magscale true def normalscale
currentpoint TR/psf$ury psfts/psf$urx psfts/psf$lly psfts/psf$llx psfts
/psf$y psfts/psf$x psfts currentpoint/psf$cy X/psf$cx X/psf$sx psf$x
psf$urx psf$llx sub div N/psf$sy psf$y psf$ury psf$lly sub div N psf$sx
psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub
TR/showpage{}N/erasepage{}N/copypage{}N/p 3 def @MacSetUp}N/doclip{
psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2
roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath
moveto}N/endTexFig{end psf$SavedState restore}N/@beginspecial{SDict
begin/SpecialSave save N gsave normalscale currentpoint TR
@SpecialDefaults count/ocount X/dcount countdictstack N}N/@setspecial{
CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto
closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx
sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR
}{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse
CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury
lineto closepath clip}if/showpage{}N/erasepage{}N/copypage{}N newpath}N
/@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{end}
repeat grestore SpecialSave restore end}N/@defspecial{SDict begin}N
/@fedspecial{end}B/li{lineto}B/rl{rlineto}B/rc{rcurveto}B/np{/SaveX
currentpoint/SaveY X N 1 setlinecap newpath}N/st{stroke SaveX SaveY
moveto}N/fil{fill SaveX SaveY moveto}N/ellipse{/endangle X/startangle X
/yrad X/xrad X/savematrix matrix currentmatrix N TR xrad yrad scale 0 0
1 startangle endangle arc savematrix setmatrix}N end

%%EndProcSet
TeXDict begin 39158280 55380996 1000 600 600
(d:\Users\argray\work\publications\Papers\1999\Errors (Metrics'99)/subjective-errors.dvi)
@start /Fa 133[29 33 1[50 1[37 21 29 29 1[37 37 37 54
21 2[21 37 37 21 33 37 33 1[37 12[42 37 1[54 46 54 50
62 2[33 25 2[46 46 54 50 1[46 7[37 37 37 5[37 3[25 3[25
25 25 39[{TeXBase1Encoding ReEncodeFont}42 74.7198 /Times-Italic
rf /Fb 105[37 27[33 37 37 54 37 37 21 29 25 1[37 37 37
58 21 37 21 21 37 37 25 33 37 33 37 33 3[25 1[25 1[54
1[71 54 54 46 42 50 1[42 1[54 66 46 54 29 25 54 54 42
46 54 50 50 54 6[21 37 37 37 37 37 37 37 37 37 37 1[19
25 19 2[25 25 37[42 2[{TeXBase1Encoding ReEncodeFont}67
74.7198 /Times-Roman rf
%DVIPSBitmapFont: Fc cmr7 7 1
/Fc 1 51 df<13FF000313E0380E03F0381800F848137C48137E00787F12FC6CEB1F80A4
127CC7FC15005C143E147E147C5C495A495A5C495A010EC7FC5B5B903870018013E0EA01
80390300030012065A001FB5FC5A485BB5FCA219267DA521>50 D
E
%EndDVIPSBitmapFont
/Fd 82[30 22[46 28[46 2[46 51 30 36 41 2[46 51 76 25
2[25 1[46 30 41 51 41 51 46 19[86 3[36 5[66 1[66 9[46
46 46 46 46 46 46 2[23 46[{TeXBase1Encoding ReEncodeFont}32
91.3242 /Times-Bold rf /Fe 82[28 51[46 46 65 46 51 28
46 32 51 51 51 51 74 23 2[23 51 51 28 46 51 46 51 46
12[51 3[55 65 9[55 3[60 7[46 46 46 46 46 46 46 46 46
46 1[23 1[23 2[28 28 37[51 2[{TeXBase1Encoding ReEncodeFont}44
83.022 /Helvetica-Bold rf
%DVIPSBitmapFont: Ff cmsy10 10 3
/Ff 3 107 df<007FB81280B912C0A26C17803204799641>0 D<126012F812FEEA7F80EA
3FE0EA0FF8EA03FEC66C7EEB3FE0EB0FF8EB03FE903800FF80EC3FE0EC0FF8EC03FE9138
00FF80ED3FE0ED0FF8ED03FE923800FF80EE3FE0EE0FF8EE03FE933800FF80EF3FC0171F
EF7F80933801FF00EE07FCEE1FF0EE7FC04B48C7FCED07FCED1FF0ED7FC04A48C8FCEC07
FCEC1FF0EC7FC04948C9FCEB07FCEB1FF0EB7FC04848CAFCEA07FCEA1FF0EA7FC048CBFC
12FC1270CCFCAE007FB81280B912C0A26C1780324479B441>21 D<126012F0B3B3B3B3A9
1260045377BD17>106 D E
%EndDVIPSBitmapFont
%DVIPSBitmapFont: Fg cmmi10 10 6
/Fg 6 115 df<EA01FCD807FF1506260E1FC0140E000C6D141CD81C0F1538261807F014
70000016E06D6CEB01C0EE03806E140001015C160E6D6C5B5E5E027F5B4B5A4B5ADA3F83
C7FC1587158EEC1F9C15F85D5D6E5A81A214074A7E141F143F1473ECE3F8EB01C3148390
380381FCEB0701130E496C7E5B5B49137F485A49800003143F48C7FC000E6E7E48163048
91380FE07048EDF06048020713E048913801FFC0C9EA7F002F367EA434>31
D<121C127FEAFF80A5EA7F00121C0909798817>58 D<163FED1FFFA3ED007F167EA216FE
A216FCA21501A216F8A21503A216F0A21507A2027E13E0903803FF8790380FC1CF90381F
00EF017EEB7FC049133F485A4848131F000715805B000F143F485A1600485A5D127F90C7
127EA215FE5A485CA21401A248ECF80CA21403161CEDF0181407007C1538007E010F1330
003E131F027B13706C01E113E03A0F83C0F9C03A03FF007F80D800FCEB1F00283B7DB92B
>100 D<EC3FC0903801FFF0903807E07890381F801C90387E001E49130E485A485A1207
485A49131E001F141C153C484813F8EC03E0007FEB3FC09038FFFE0014E090C8FC5A5AA7
007E140315071506003E140E153C6C14706C6C13E0EC07C03903E03F003801FFF838003F
C020267DA427>I<90390F8003F090391FE00FFC903939F03C1F903A70F8700F80903AE0
FDE007C09038C0FF80030013E00001491303018015F05CEA038113015CA2D800031407A2
5CA20107140FA24A14E0A2010F141F17C05CEE3F80131FEE7F004A137E16FE013F5C6E48
5A4B5A6E485A90397F700F80DA383FC7FC90387E1FFCEC07E001FEC9FCA25BA21201A25B
A21203A25B1207B512C0A32C3583A42A>112 D<3903E001F83907F807FE390E3C1E0739
1C3E381F3A183F703F800038EBE07F0030EBC0FF00705B00601500EC007E153CD8E07F90
C7FCEAC07EA2120013FE5BA312015BA312035BA312075BA3120F5BA3121F5B0007C9FC21
267EA425>114 D E
%EndDVIPSBitmapFont
%DVIPSBitmapFont: Fh cmr10 10 14
/Fh 14 62 df<146014E0EB01C0EB0380EB0700130E131E5B5BA25B485AA2485AA21207
5B120F90C7FCA25A121EA2123EA35AA65AB2127CA67EA3121EA2121F7EA27F12077F1203
A26C7EA26C7E1378A27F7F130E7FEB0380EB01C0EB00E01460135278BD20>40
D<12C07E12707E7E7E120F6C7E6C7EA26C7E6C7EA21378A2137C133C133E131EA2131F7F
A21480A3EB07C0A6EB03E0B2EB07C0A6EB0F80A31400A25B131EA2133E133C137C1378A2
5BA2485A485AA2485A48C7FC120E5A5A5A5A5A13527CBD20>I<15301578B3A6007FB812
F8B912FCA26C17F8C80078C8FCB3A6153036367BAF41>43 D<EB03F8EB1FFF90387E0FC0
9038F803E03901E000F0484813780007147C48487FA248C77EA2481580A3007EEC0FC0A6
00FE15E0B3007E15C0A4007F141F6C1580A36C15006D5B000F143EA26C6C5B6C6C5B6C6C
485A6C6C485A90387E0FC0D91FFFC7FCEB03F8233A7DB72A>48 D<EB01C013031307131F
13FFB5FCA2131F1200B3B3A8497E007FB512F0A31C3879B72A>I<EB0FF0EB7FFE48B57E
3903E03FE0390F000FF0000E6D7E486D7E486D7E123000706D7E126012FCB4EC7F807FA5
6CC7FC121CC8FCEDFF00A34A5A5D14035D4A5A5D140F4A5A4A5A92C7FC147C5C495A495A
495A495A91C8FC011EEB01805B5B49130348481400485A485A000EC75A000FB6FC5A5A48
5CB6FCA321387CB72A>I<EB07F8EB3FFF4913C03901F80FF03903C007F848486C7E380E
0001000F80381FE0006D7FA56C5A6C5AC85A1401A25D4A5AA24A5A5DEC0F80027EC7FCEB
1FFCECFF809038000FE06E7EEC01FC816E7EED7F80A216C0A2153F16E0A2121EEA7F8048
7EA416C049137F007F1580007EC7FC0070ECFF006C495A121E390F8003F83907F00FF000
01B512C06C6C90C7FCEB0FF8233A7DB72A>I<1538A2157815F8A2140114031407A2140F
141F141B14331473146314C313011483EB030313071306130C131C131813301370136013
C01201EA038013005A120E120C5A123812305A12E0B712F8A3C73803F800AB4A7E0103B5
12F8A325397EB82A>I<0006140CD80780133C9038F003F890B5FC5D5D158092C7FC14FC
38067FE090C9FCABEB07F8EB3FFE9038780F803907E007E090388003F0496C7E12066E7E
C87EA28181A21680A4123E127F487EA490C71300485C12E000605C12700030495A00385C
6C1303001E495A6C6C485A3907E03F800001B5C7FC38007FFCEB1FE0213A7CB72A>I<EC
3FC0903801FFF0010713FC90380FE03E90383F800790387E001F49EB3F804848137F485A
A2485A000FEC3F0049131E001F91C7FCA2485AA3127F90C9FCEB01FC903807FF8039FF1E
07E090383801F0496C7E01607F01E0137E497FA249148016C0151FA290C713E0A57EA56C
7E16C0A2121FED3F807F000F15006C6C5B15FE6C6C5B6C6C485A3900FE07F090383FFFC0
6D90C7FCEB03FC233A7DB72A>I<12301238123E003FB612E0A316C05A168016000070C7
12060060140E5D151800E01438485C5D5DC712014A5A92C7FC5C140E140C141C5CA25CA2
14F0495AA21303A25C1307A2130FA3495AA3133FA5137FA96DC8FC131E233B7BB82A>I<
EB03F8EB1FFF017F13C09038FC07F03901E001F848486C7E4848137C90C77E48141E000E
141F001E80A3121FA27F5D01E0131E6C6C133E01FC133C6D5B6C6C6C5AECC1E06CEBF3C0
6C01FFC7FC6C5BEB3FFF6D13C081017F13F801F07F3903E07FFE3907801FFF48486C1380
481303003E6D13C0003CEB007F007C143F0078EC0FE000F814075A1503A21501A36C15C0
12781503007C15806CEC07006C5C6C6C131ED807E0137C3903F803F0C6B55A013F1380D9
07FCC7FC233A7DB72A>I<EB03F8EB1FFF017F13C09038FC07E03903F803F048486C7E48
486C7E49137E121F48487FA2007F158090C7FCA248EC1FC0A616E0A56C143FA27F123F00
1F147FA26C6C13FF6C6C13DF000313013901F0039F3900FC0F1FD93FFC13C0EB07F090C7
FC153F1680A316005D000F147E487E486C5BA24A5A4A5A49485A6C48485A001C495A260F
807FC7FC3807FFFC000113F038003FC0233A7DB72A>I<007FB812F8B912FCA26C17F8CC
FCAE007FB812F8B912FCA26C17F836167B9F41>61 D E
%EndDVIPSBitmapFont
%DVIPSBitmapFont: Fi cmti10 10 23
/Fi 23 123 df<15181538157815F0140114031407EC0FE0141F147FEB03FF90383FEFC0
148FEB1C1F13001580A2143FA21500A25CA2147EA214FEA25CA21301A25CA21303A25CA2
1307A25CA2130FA25CA2131FA25CA2133FA291C7FC497EB61280A31D3877B72A>49
D<EC03F8EC0FFE91383C0F809138F007C0903901E003E0D903C013F09038078001020013
F8130E131E90391C6000FCEB3870EC30011370A213F013E0EC70030001016013F813C014
E0ECC00701C114F0903881800F018314E09039C7001FC001FEEB3F80D80078140090C712
7E5D4A5A4A5AEC07C0EC1F80023EC7FC14FCEB01F0EB07C0495A011EC8FC137C4914C048
4813015B485A4848130348C71380000E1407001E140F48EC1F00D83FF85B397FFFC07E39
783FFFFCEA700FD8F0075BD8E0015B6D13C0021FC7FC263A79B72A>I<0107B8FCA3903A
000FF000034BEB007F183E141F181E5DA2143FA25D181C147FA29238000380A24A130718
004A91C7FC5E13015E4A133E167E49B512FEA25EECF8000107147C163C4A1338A2010F14
7818E04A13701701011F16C016004A14031880013F150718004A5CA2017F151E173E91C8
123C177C4915FC4C5A4914070001ED7FF0B8FCA25F38397BB838>69
D<0107B512FCA25E9026000FF8C7FC5D5D141FA25DA2143FA25DA2147FA292C8FCA25CA2
5CA21301A25CA21303A25CA21307A25CA2130F170C4A141CA2011F153C17384A1478A201
3F157017F04A14E01601017F140317C091C71207160F49EC1F80163F4914FF0001020713
00B8FCA25E2E397BB834>76 D<ED03FE92383FFFC09238FC07F0913903E001F891390F80
007C023FC77E027E8002F815804948EC0FC0EB07E04948EC07E0131F4A15F049C8120313
7E01FE16F8485AA2485AA2485AA2120F5B001F16075B123FA34848ED0FF0A448C9EA1FE0
A3EF3FC0A21880177F18005F5F16015F6C4B5A4C5AA24C5A6C4B5A6D4A5A001F93C7FC6D
147E000F5D6C6CEB03F06C6C495A6C6CEB0F806C6C013FC8FC90383F01FC90381FFFE001
0190C9FC353D74BA40>79 D<92383FC00E913901FFF01C020713FC91391FC07E3C91393F
001F7C027CEB0FF84A130749481303495A4948EB01F0A2495AA2011F15E091C7FCA34915
C0A36E90C7FCA2806D7E14FCECFF806D13F015FE6D6D7E6D14E0010080023F7F14079138
007FFC150F15031501A21500A2167C120EA3001E15FC5EA3003E4A5AA24B5AA2007F4A5A
4B5A6D49C7FC6D133ED8F9F013FC39F8FC03F839F07FFFE0D8E01F138026C003FCC8FC2F
3D7ABA2F>83 D<0007B812E0A25AD9F800EB001F01C049EB07C0485AD900011403121E00
1C5C003C17801403123800785C00701607140700F01700485CA2140FC792C7FC5DA2141F
A25DA2143FA25DA2147FA292C9FCA25CA25CA21301A25CA21303A25CA21307A25CA2130F
A25CEB3FF0007FB512F8B6FCA2333971B83B>I<003FB539800FFFFEA326007F80C7EA7F
8091C8EA3F00173E49153CA2491538A20001167817705BA2000316F05F5BA2000715015F
5BA2000F15035F5BA2001F150794C7FC5BA2003F5D160E5BA2007F151E161C90C8FCA216
3C4815385A16781670A216F04B5A5E1503007E4A5A4BC8FC150E6C143E6C6C5B15F0390F
C003E03907F01FC00001B5C9FC38007FFCEB1FE0373B70B83E>I<ED01F815FFA3150316
F0A21507A216E0A2150FA216C0A2151FA21680A2153FA202F81300EB07FE90381F877F90
383E03FF017C5BEBF80112013803F00048485B120FEBC001121F5DEA3F801403127F0100
5BA214075A485CA2140FA248ECC1C0A2141F15C3ED8380143F1587007C017F1300ECFF07
6C485B9038038F8E391F0F079E3907FE03FC3901F000F0253B77B92A>100
D<147F903803FFC090380FC1E090383F00F0017E13785B485A485A485A120F4913F8001F
14F0383F8001EC07E0EC1F80397F81FF00EBFFF891C7FC90C8FC5A5AA55AA21530007C14
381578007E14F0003EEB01E0EC03C06CEB0F806CEB3E00380781F83803FFE0C690C7FC1D
2677A426>I<EC07C0EC3FF09138FC38E0903901F01FF0EB03E0903807C00FEB0F80011F
1307D93F0013E05B017E130F13FE4914C01201151F1203491480A2153F1207491400A25D
A249137EA215FEA25D00031301140314076C6C485A0000131FEB787BEB3FF390380FC3F0
EB00031407A25DA2140F5D121C007E131F5D00FE49C7FC147E5C387801F8387C07E0381F
FF80D803FEC8FC24367CA426>103 D<EB01C0EB07E014F0130F14E01307EB038090C7FC
AB13F0EA03FCEA071EEA0E1F121CA212385B1270A25BEAF07E12E013FEC65AA212015B12
03A25B12075BA2000F13E013C013C1001F13C01381A2EB83801303EB0700A2130E6C5AEA
07F8EA01E0143879B619>105 D<EB03F0EA01FFA3EA00075CA3130F5CA3131F5CA3133F
91C8FCA35B017EEB0F80ED3FE015F09039FE01C1F09038FC0387EC0707140E0001011C13
E0EBF83891383003800270C7FC00035BEBF1C0EBF38001FFC8FCEA07FC7FEBFFC0EBE7F8
380FE1FCEBC07E147F80001F809039801F81C0A21583003F013F138001001303A2150748
1500007E133EEC1E0E151E00FE6D5A48EB07F80038EB01E0243B7BB926>107
D<EB0FC0EA07FFA3EA001F1480A2133FA21400A25BA2137EA213FEA25BA21201A25BA212
03A25BA21207A25BA2120FA25BA2121FA25BA2123FA290C7FCA25AA2EA7E0EA212FE131E
EAFC1CA2133C133812F81378EA7870EA7CE0121FEA0F80123B79B915>I<D801E013FE3A
07F803FF803A0E3C0F07C03A1E3E3C03E0261C1F787F39383FF00114E0007813C0007081
14804A485AEAF07FEAE07EA20000140701FE5C5BA2150F00015D5B151F5E12034990383F
8380160316070007027F130049137EA2160E000F147C49141E161C5E001FEC3C7849EB1F
E00007C7EA0780292679A42F>110 D<147F903803FFC090380FC1F090381F00F8017E13
7C5B4848137E4848133E0007143F5B120F485AA2485A157F127F90C7FCA215FF5A4814FE
A2140115FC5AEC03F8A2EC07F015E0140F007C14C0007EEB1F80003EEB3F00147E6C13F8
380F83F03803FFC0C648C7FC202677A42A>I<9039078007C090391FE03FF090393CF078
7C903938F8E03E9038787FC00170497EECFF00D9F0FE148013E05CEA01E113C15CA2D800
03143FA25CA20107147FA24A1400A2010F5C5E5C4B5A131F5EEC80035E013F495A6E485A
5E6E48C7FC017F133EEC70FC90387E3FF0EC0F8001FEC9FCA25BA21201A25BA21203A25B
1207B512C0A3293580A42A>I<3903C003F0390FF01FFC391E783C0F381C7C703A3C3EE0
3F8038383FC0EB7F800078150000701300151CD8F07E90C7FCEAE0FE5BA2120012015BA3
12035BA312075BA3120F5BA3121F5BA3123F90C9FC120E212679A423>114
D<14FE903807FF8090380F83C090383E00E04913F00178137001F813F00001130313F0A2
15E00003EB01C06DC7FC7FEBFFC06C13F814FE6C7F6D13807F010F13C01300143F141F14
0F123E127E00FE1480A348EB1F0012E06C133E00705B6C5B381E03E06CB45AD801FEC7FC
1C267AA422>I<EB0380EB07C0130FA4131F1480A3133F1400A35B137E007FB5FCA2B6FC
3800FC00A312015BA312035BA312075BA3120F5BA3121FEB801CA2143C003F1338EB0078
147014F014E0EB01C0EA3E03381F0780380F0F00EA07FCEA01F0183579B31C>I<01F013
0ED803FC133FD8071EEB7F80EA0E1F121C123C0038143F49131F0070140FA25BD8F07E14
0000E08013FEC6485B150E12015B151E0003141C5BA2153C000714385B5DA35DA24A5A14
0300035C6D48C7FC0001130E3800F83CEB7FF8EB0FC0212679A426>118
D<13F0D803FCEB01C0D8071EEB03E0D80E1F1307121C123C0038140F4914C01270A24913
1FD8F07E148012E013FEC648133F160012015B5D0003147E5BA215FE00075C5BA214015D
A314035D14070003130FEBF01F3901F87FE038007FF7EB1FC7EB000F5DA2141F003F5C48
133F92C7FC147E147C007E13FC387001F8EB03E06C485A383C1F80D80FFEC8FCEA03F023
3679A428>121 D<903903C0038090380FF007D91FF81300496C5A017F130E9038FFFE1E
9038F83FFC3901F007F849C65A495B1401C7485A4A5A4AC7FC141E5C5C5C495A495A495A
49C8FC131E5B49131C5B4848133C48481338491378000714F8390FF801F0391FFF07E038
3E1FFFD83C0F5B00785CD8700790C7FC38F003FC38E000F021267BA422>I
E
%EndDVIPSBitmapFont
/Fj 105[42 1[37 37 24[37 42 42 60 42 42 23 32 28 42 42
42 42 65 23 42 23 23 42 42 28 37 42 37 42 37 3[28 1[28
3[78 60 60 51 46 55 1[46 60 60 74 51 60 1[28 60 60 46
51 60 55 55 60 3[47 2[23 42 42 42 42 42 42 42 42 42 42
23 21 28 21 47 42 28 28 28 1[69 33[46 46 2[{
TeXBase1Encoding ReEncodeFont}76 83.022 /Times-Roman
rf /Fk 133[32 37 37 55 37 42 23 32 32 42 42 42 42 60
23 37 23 23 42 42 23 37 42 37 42 42 12[46 42 3[60 1[69
46 2[28 3[51 60 4[42 11[42 42 42 2[21 28 21 2[28 28 37[42
2[{TeXBase1Encoding ReEncodeFont}44 83.022 /Times-Italic
rf /Fl 134[50 3[55 33 39 44 1[55 50 55 83 28 1[33 28
55 50 33 44 55 44 55 50 11[72 2[72 4[94 66 2[39 78 2[66
1[72 1[72 7[50 50 50 50 50 50 50 50 50 3[33 45[{
TeXBase1Encoding ReEncodeFont}39 99.6264 /Times-Bold
rf
%DVIPSBitmapFont: Fm cmsy10 12 2
/Fm 2 104 df<ED0FE015FF913803FC00EC0FE0EC3FC04A5A4AC7FC5C495AA2495AB3AD
495AA2495A131F495A495A01FEC8FCEA07F8EAFFE0A2EA07F8EA00FEEB7F806D7E6D7E13
0F6D7EA26D7EB3AD6D7EA26D7E806E7E6E7EEC0FE0EC03FC913800FFE0150F236479CA32
>102 D<12FEEAFFE0EA07F8EA00FEEB7F806D7E6D7E130F6D7EA26D7EB3AD6D7EA26D7E
806E7E6E7EEC0FE0EC03FC913800FFE0A2913803FC00EC0FE0EC3FC04A5A4AC7FC5C495A
A2495AB3AD495AA2495A131F495A495A01FEC8FCEA07F8EAFFE048C9FC236479CA32>I
E
%EndDVIPSBitmapFont
%DVIPSBitmapFont: Fn cmsy8 8 1
/Fn 1 122 df<1338137CA81338A7007C137CB512FEA3387C387C00001300A5137CB3A4
1338AD173D7CAE20>121 D E
%EndDVIPSBitmapFont
/Fo 133[44 50 1[72 50 50 28 39 33 1[50 50 50 78 28 50
1[28 50 50 33 44 50 44 50 44 11[72 1[55 66 2[72 1[89
2[39 33 1[72 2[72 66 66 72 92 17[25 1[25 44[{
TeXBase1Encoding ReEncodeFont}38 99.6264 /Times-Roman
rf /Fp 134[60 60 86 60 66 40 47 53 1[66 60 66 100 33
1[40 33 66 60 40 53 66 53 66 60 26[73 11[40 58[{
TeXBase1Encoding ReEncodeFont}25 119.552 /Times-Bold
rf end
%%EndProlog
%%BeginSetup
%%Feature: *Resolution 600dpi
TeXDict begin
%%PaperSize: A4

%%EndSetup
%%Page: 1 1
1 0 bop -142 457 a Fp(F)m(actors)29 b(systematically)f(associated)h
(with)i(err)n(ors)e(in)i(subjecti)o(v)o(e)e(estimates)g(of)g(softwar)n
(e)558 607 y(de)n(v)o(elopment)i(eff)m(ort:)36 b(the)30
b(stability)g(of)f(expert)i(judgment)564 922 y Fo(Andre)n(w)24
b(R.)h(Gray)-6 b(,)25 b(Stephen)g(G.)f(MacDonell,)g(and)2400
886 y Fn(y)2436 922 y Fo(Martin)g(J.)g(Shepperd)752 1039
y(Department)g(of)h(Information)f(Science,)h(Uni)n(v)o(ersity)e(of)i
(Otago)817 1120 y Fn(y)852 1156 y Fo(Department)g(of)g(Computing,)e
(Bournemouth)h(Uni)n(v)o(ersity)308 1273 y Fm(f)p Fo(agray)p
Fm(gf)p Fo(ste)n(v)o(emac)p Fm(g)p Fo(@infoscience.otago.ac.nz)2169
1237 y Fn(y)2204 1273 y Fo(mshepper@bournemouth.ac.uk)617
1640 y Fl(Abstract)-83 1846 y Fk(Softwar)m(e)16 b(metric-based)f
(estimation)h(of)g(pr)l(oject)g(de)o(velopment)-182 1946
y(ef)o(fort)32 b(is)i(most)f(often)g(performed)f(by)h(e)n(xpert)g
(judgment)e(r)o(ather)-182 2045 y(than)20 b(by)i(using)e(an)i
(empirically)f(derived)g(model)g(\(although)d(suc)o(h)-182
2145 y(may)31 b(be)g(used)g(by)g(the)g(e)n(xpert)h(to)f(assist)h(their)
g(decision\).)57 b(One)-182 2245 y(question)16 b(that)i(can)f(be)h(ask)
o(ed)g(about)f(these)h(estimates)g(is)h(how)f(sta-)-182
2344 y(ble)25 b(ar)m(e)i(the)n(y)e(with)h(r)m(espect)g(to)g(c)o(har)o
(acteristics)g(of)g(the)g(de)o(velop-)-182 2444 y(ment)h(pr)l(ocess)h
(and)e(pr)l(oduct?)45 b(This)28 b(stability)f(can)g(be)g(assessed)-182
2543 y(in)22 b(r)m(elation)g(to)h(the)f(de)m(gr)m(ee)g(to)h(whic)o(h)f
(the)g(pr)l(oject)h(has)f(advanced)-182 2643 y(o)o(ver)i(time)o(,)i
(the)f(type)f(of)h(module)e(for)i(whic)o(h)g(the)f(estimate)h(is)g(be-)
-182 2743 y(ing)k(made)o(,)i(and)e(the)h(c)o(har)o(acteristics)f(of)h
(that)g(module)o(.)52 b(In)29 b(this)-182 2842 y(paper)23
b(we)h(e)n(xamine)g(a)g(set)g(of)g(e)n(xpert-derived)f(estimates)i(for)
f(the)-182 2942 y(ef)o(fort)31 b(r)m(equir)m(ed)f(to)h(de)o(velop)f(a)h
(collection)f(of)h(modules)f(fr)l(om)i(a)-182 3042 y(lar)m(g)o(e)23
b(health-car)m(e)e(system.)35 b(Statistical)23 b(tests)h(ar)m(e)f(used)
g(to)h(iden-)-182 3141 y(tify)31 b(r)m(elationships)e(between)i(the)f
(type)g(\(scr)m(een)h(or)g(r)m(eport\))f(and)-182 3241
y(c)o(har)o(acteristics)e(of)i(modules)e(and)g(the)i(lik)o(elihood)e
(of)h(the)g(asso-)-182 3340 y(ciated)i(de)o(velopment)f(ef)o(fort)j
(being)e(under)n(-estimated,)i(appr)l(ox-)-182 3440 y(imately)e(corr)m
(ect,)j(or)d(o)o(ver)n(-estimated.)57 b(Distinct)32 b(r)m(elationships)
-182 3540 y(ar)m(e)20 b(found)e(that)i(sug)o(g)o(est)g(that)g(the)g
(estimation)f(pr)l(ocess)i(being)e(e)n(x-)-182 3639 y(amined)g(was)i
(not)f(unbiased)e(to)i(suc)o(h)g(c)o(har)o(acteristics.)-182
3992 y Fl(1)99 b(Intr)n(oduction)-83 4229 y Fj(The)25
b(ability)h(to)f(accurately)f(predict)h(the)g(de)n(v)o(elopment)e(ef)n
(fort)-182 4329 y(required)31 b(for)h(softw)o(are)h(systems)h(and)e
(their)h(constituent)f(mod-)-182 4428 y(ules,)25 b(as)g(early)f(as)h
(possible)f(in)h(the)g(de)n(v)o(elopment)c(process,)k(is)g(of)-182
4528 y(considerable)30 b(importance)g(to)j(an)o(y)e(or)o(ganization)e
(carrying)h(out)-182 4628 y(softw)o(are)22 b(de)n(v)o(elopment.)30
b(Accurac)o(y)22 b(for)g(softw)o(are)g(metric)h(mod-)-182
4727 y(els)32 b(may)e(of)h(course)g(be)g(de\002ned)f(in)h(man)o(y)f
(dif)n(ferent)g(w)o(ays)h(de-)-182 4827 y(pending)d(on)h(the)h(cost)h
(function)d(for)h(errors.)54 b(Commonly)28 b(used)-182
4926 y(measurements)d(of)i(softw)o(are)g(metric)g(model)f(goodness)g
(include)-182 5026 y(absolute)i(and)g(relati)n(v)o(e)f(errors)h(as)i
(well)f(as)g(threshold)e(measures)-182 5126 y(such)c(as)i
Fi(pr)l(e)l(d)9 b Fh(\(\))q Fj(.)36 b(The)24 b(optimal)f(measurement)f
(of)i(model)f(good-)-182 5225 y(ness)h(depends)e(on)h(both)g(project)g
(and)g(or)o(ganizational)d(character)n(-)-182 5325 y(istics)h(which)f
(can)g(v)n(ary)f(widely)-5 b(.)2073 1640 y(In)16 b(this)g(analysis)g(a)
g(simpli\002cation)g(of)f(the)h(widely)f(used)h Fi(pr)l(e)l(d)9
b Fh(\(\))1974 1739 y Fj(measure)32 b(is)h(used,)j(where)c(a)h
(successful)g(estimation)f(is)h(within)1974 1839 y(35\045)15
b(of)h(the)f(actual,)h(and)f(under)n(-)f(and)h(o)o(v)o(er)n(-estimates)
g(are)g(de\002ned)1974 1939 y(accordingly)-5 b(.)34 b(This)25
b(allo)n(ws)f(for)g(the)g(use)h(of)f(classi\002cation)g(mod-)1974
2038 y(els)i(to)f(predict)f(types)h(of)g(estimate)g(errors.)39
b(De)n(v)o(elopment)22 b(ef)n(fort)1974 2138 y(predictions)h(play)g(an)
h(important)f(role)h(in)g(both)f(project)h(manage-)1974
2238 y(ment)h(and)f(higher)n(-le)n(v)o(el)f(management)g(decisions.)39
b(These)25 b(deci-)1974 2337 y(sions)g(include)f(acceptance/rejection)e
(of)j(projects,)g(resource)f(al-)1974 2437 y(location,)j(and)g
(planning,)g(monitoring,)f(and)g(controlling)f(de)n(v)o(el-)1974
2537 y(opment.)2073 2669 y(Predictions)f(of)f(de)n(v)o(elopment)e(ef)n
(fort)i(can)h(be)g(made)f(in)h(terms)1974 2769 y(of)31
b(person-hours)d(\(or)i(days,)j(months,)g(or)d(years)h(depending)e(on)
1974 2868 y(the)c(scope)g(of)f(the)i(project\))d(using)i(system/module)
e(characteris-)1974 2968 y(tics)i(commonly)d(referred)g(to)i(as)h
(softw)o(are)f(metrics.)37 b(These)24 b(may)1974 3067
y(include)29 b(assessments)i(of)e(module)g(size)h(and)g(comple)o(xity)
-5 b(,)30 b(each)1974 3167 y(of)h(which)f(can)h(be)g(measured)f(using)g
(man)o(y)g(dif)n(ferent)g(softw)o(are)1974 3267 y(metrics,)k(including)
c(those)i(a)n(v)n(ailable)f(from)g(the)h(design)f(phase.)1974
3366 y(The)19 b(lo)n(wer)h(the)f(le)n(v)o(el)h(of)f(detail)h(of)g(the)f
(required)f(information)g(for)1974 3466 y(a)28 b(predicti)n(v)o(e)d
(model)i(\(i.e.)46 b(the)27 b(earlier)g(the)h(model)e(can)h(be)g(used)
1974 3566 y(with)f(an)f(acceptable)f(de)o(gree)g(of)i(accurac)o(y\),)e
(the)i(more)e(v)n(aluable)1974 3665 y(the)f(model)g(will)h(be)g(since)f
(it)i(is)f(in)g(the)f(early)g(stages)h(of)f(de)n(v)o(elop-)1974
3765 y(ment)d(that)g(such)g(estimates)h(are)f(most)g(crucial.)2073
3897 y(When)g(estimates)g(of)f(ef)n(fort)g(are)h(made,)e(using)h(whate)
n(v)o(er)g(tech-)1974 3997 y(nique)38 b(\(such)g(as)h(e)o(xpert)e
(opinion,)42 b(re)o(gression)37 b(models,)42 b(case-)1974
4097 y(based)23 b(reasoning,)g(neural)f(netw)o(ork)h(models,)g(fuzzy)g
(logic)g(mod-)1974 4196 y(els,)17 b(and)d(so)i(on\),)f(some)g(subjecti)
n(vity)f(is)i(usually)f(in)m(v)n(olv)o(ed)e(\226)j(either)1974
4296 y(in)26 b(making)f(the)h(estimates)h(themselv)o(es)e(\(since)h
(man)o(y)f(estimates)1974 4396 y(are)c(in)g(f)o(act)h(simply)f
Fk(guestimates)f Fj(based)h(on)g(subjecti)n(v)o(e)f(opinion\))1974
4495 y(or)26 b(in)h(calibrating)e(some)i(inputs)f(into)g(the)h(model)e
(such)i(as)g(com-)1974 4595 y(ple)o(xity)19 b(f)o(actors)h(in)g
(function)f(point)g(analysis)h(\(FP)-8 b(A\))21 b([1)o(].)2073
4727 y(FP)-8 b(A)26 b(can)d(be)h(subjecti)n(v)o(e)f(in)h(tw)o(o)g(main)
g(w)o(ays.)36 b(First,)26 b(a)e(count)1974 4827 y(must)h(be)h(made)f
(of)g(the)g(v)n(arious)g(functions,)g(which)g(are)g(e)o(xternal)1974
4926 y(inputs,)20 b(e)o(xternal)g(outputs,)g(internal)g(logical)h
(\002les,)h(e)o(xternal)e(logi-)1974 5026 y(cal)j(\002les,)h(and)d(e)o
(xternal)h(inquiries.)31 b(These)22 b(functions)f(are)h(classi-)1974
5126 y(\002ed)c(as)h(either)f(simple,)h(a)n(v)o(erage,)e(or)h(comple)o
(x.)k(These)d(counts)e(are)1974 5225 y(then)e(used)g(to)h(construct)e
(a)i(weighted)e(count,)h(with)h(the)f(weightings)1974
5325 y(ob)o(viously)i(dependent)f(on)j(comple)o(xity)d(\(the)j(weights)
f(depend)g(on)p eop
%%Page: 2 2
2 1 bop -182 83 a Fj(the)20 b(function)e(type)h(and)g(range)g(from)g(3)
h(to)g(15\).)k(Second,)19 b(the)h(total)-182 183 y(count)27
b(for)h(a)g(system)h(is)g(then)f(multiplied)f(by)h(a)h(subjecti)n(v)o
(e)e(pro-)-182 282 y(cessing)18 b(comple)o(xity)e(score)i(constructed)e
(from)h(rating)h(14)f(f)o(actors)-182 382 y(on)k(a)i(0)g(to)f(5)g
(scale)h(\(from)e(0)h(being)g(not)g(present)f(or)h(no)g(in\003uence,)
-182 482 y(through)c(to)i(5)g(being)f(strong)h(in\003uence)f
(throughout\).)-83 581 y(Gi)n(v)o(en)31 b(the)h(scarcity)f(of)h(data)f
(for)h(calibrating)e(such)i(models)-182 681 y(some)27
b(subjecti)n(vity)f(is)i(ine)n(vitable,)g(and)f(probably)e(desirable.)
45 b(If)-182 780 y(the)17 b(estimation)g(process)g(w)o(as)h(limited)f
(to)h(only)e(empirical)h(models)-182 880 y(then)23 b(the)h(small)h
(data)e(sets,)j(coupled)c(with)j(a)f(high)f(proportion)e(of)-182
980 y(outliers,)27 b(w)o(ould)e(be)h(unlik)o(ely)f(to)i(result)f(in)g
(generalisable)f(mod-)-182 1079 y(els.)54 b(The)30 b(inclusion)f(of)g
(subjecti)n(v)o(e)g(elements)h(in)g(such)g(models)-182
1179 y(allo)n(ws)c(for)f(a)h(great)f(reduction)f(in)i(the)g(number)e
(of)h(v)n(ariables,)h(as)-182 1279 y(well)f(as)g(the)g(accounting)d
(for)i(f)o(actors)g(that)h(are)g(dif)n(\002cult)f(to)g(mea-)-182
1378 y(sure.)g(Expert)19 b(opinion)f(is)j(often)e(dif)n(\002cult)g(to)h
(quantify)e(b)n(ut)h(can)h(be)-182 1478 y(an)g(ef)n(fecti)n(v)o(e)e
(estimating)i(tool,)f(in)h(its)h(o)n(wn)f(right)f(or)h(as)g(an)g
(adjust-)-182 1577 y(ing)f(f)o(actor)h(for)g(algorithmic)e(models)i
([13)n(].)-83 1677 y(This)h(realization)e(has)h(recently)f(prompted)f
(researchers)h(to)h(re-)-182 1777 y(visit)32 b(their)f(approach)e(to)j
(the)f(issue)h(of)f(estimation.)58 b(Since)32 b(the)-182
1876 y(1970s)f(w)o(ork)h(has)h(lar)o(gely)e(been)g(concentrated)g(on)h
(de)n(v)o(eloping)-182 1976 y(algorithmic)c(estimation)i(models,)h
(under)e(the)h(assumption)f(that)-182 2076 y(this)18
b(w)o(ould)g(\(by)g(its)h(v)o(ery)e(nature\))g(result)h(in)h(impro)o(v)
o(ements)c(in)k(es-)-182 2175 y(timation)f(accurac)o(y)-5
b(.)22 b(Whilst)f(some)d(models)h(ha)n(v)o(e)f(indeed)g(pro)o(v)o(ed)
-182 2275 y(to)k(be)g(useful)g(in)g(speci\002c)g(cases,)h(their)f
(general)f(applicability)g(re-)-182 2374 y(mains)j(uncertain.)36
b(In)24 b(a)g(study)g(of)g(the)g(causes)h(of)f(estimation)g(er)n(-)-182
2474 y(ror)g([9)o(])h(found)e(that,)j(whilst)f(subjecti)n(v)o(e)e
(guesses)i(indeed)f(tended)-182 2574 y(to)17 b(result)g(in)g
(inaccurate)e(estimates,)j(accurac)o(y)d(w)o(as)j(not)f(impro)o(v)o(ed)
-182 2673 y(with)30 b(the)g(use)h(of)f(algorithmic)e(models.)54
b(There)30 b(are)g(other)f(rea-)-182 2773 y(sons)23 b(why)g(some)h
(practitioners)e(choose)h(not)g(to)h(use)g(algorithmic)-182
2873 y(models,)18 b(as)i(illustrated)e(by)h(the)f(follo)n(wing)g(quote)
g(tak)o(en)g(from)g(the)-182 2972 y(estimation)h(methodology)e(of)j(a)h
(major)e(softw)o(are)h(house:)25 3112 y(The)15 b(current)g
([estimation])f(methodology)f(is)j(f)o(ar)g(too)25 3212
y(comple)o(x)28 b(and)h(dif)n(\002cult)g(to)h(use.)54
b(This)30 b(is)g(demon-)25 3311 y(strated)e(by)g(the)f(f)o(act)i(that)f
(it)g(is)h(not)f(used.)48 b(In)27 b(f)o(act,)25 3411
y(each)39 b(person)f(undertaking)e(estimation)i(tends)h(to)25
3511 y(utilize)20 b(their)g(o)n(wn)g(heuristic)g(approach.)-182
3650 y(Such)f(sentiments)h(are)g(also)g(partly)f(behind)f(the)i(demand)
e(for)i(sim-)-182 3750 y(pli\002ed)h(v)o(ersions)g(of)h(techniques)f
(such)h(as)g(function)f(point)g(analy-)-182 3850 y(sis,)j(as)f(these)g
(are)g(seen)g(by)f(some)h(to)g(be)f(unnecessarily)g(comple)o(x)-182
3949 y(and/or)f(detailed)h(\(e.g.)31 b(see)23 b([10)n(]\).)32
b(F)o(or)22 b(instance,)h(another)e(major)-182 4049 y(softw)o(are)e
(house)h(states:)25 4189 y(Because)33 b(we)h(can)f(write)g(programs)e
(and)h(de)n(v)o(elop)25 4288 y(systems)i(v)o(ery)e(quickly)f
Fg(:)14 b(:)g(:)48 b Fj(it)34 b(is)g(important)d(that)25
4388 y(an)o(y)16 b(estimating)g(techniques)g(we)h(use)g(are)f(quick)g
(and)25 4488 y(easy)25 b(to)g(apply)-5 b(,)25 b(otherwise)f(we)i(could)
e(end)g(up)h(tak-)25 4587 y(ing)f(as)h(long)f(to)g(estimate)h(as)g(we)g
(tak)o(e)f(to)h(write)f(the)25 4687 y(programs.)-182
4827 y(Sometimes)41 b(the)h(choice)f(of)h(method)e(is)j(imposed)e(\226)
h([5)o(])g(de-)-182 4926 y(scribe)27 b(the)g(situation)g(where)g(an)g
(or)o(ganization)d(mak)o(es)k(a)g(signif-)-182 5026 y(icant)c(change)e
(to)j(its)g(softw)o(are)e(process,)i(rendering)d(algorithmic)-182
5126 y(models)e(de)n(v)o(eloped)e(and)j(calibrated)f(from)g(historical)
g(data)h(virtu-)-182 5225 y(ally)15 b(w)o(orthless.)24
b(Under)14 b(these)i(circumstances)f(e)o(xpert)f(estimation)-182
5325 y(may)19 b(be)h(the)h(only)e(alternati)n(v)o(e.)2073
83 y(As)j(a)g(result)f(we)h(ha)n(v)o(e)e(be)o(gun)g(to)h(see)h(a)g
(resur)o(gence)c(in)k(the)f(use)1974 183 y(of)k(informed)e(subjecti)n
(v)o(e)h(estimation)g(as)i(a)f(realistic)h(alternati)n(v)o(e.)1974
282 y(In)31 b(a)h(comparati)n(v)o(e)d(study)i(of)g(case-based)g
(reasoning,)h(function)1974 382 y(point)d(analysis)h(COCOMO)i([2)o(],)g
(and)e(e)o(xpert)f(estimation,)g([11)o(])1974 482 y(found)17
b(that)h(the)h(e)o(xpert)e(out-performed)e(all)k(the)g(other)f
(techniques)1974 581 y(in)25 b(terms)g(of)g(accurac)o(y)e(and)i
(consistenc)o(y)-5 b(.)38 b(Moreo)o(v)o(er)m(,)23 b(the)i(case-)1974
681 y(based)h(reasoning)f(method,)i(which)f(attempts)h(to)g(mimic)f
(the)h(e)o(x-)1974 780 y(pert)20 b(estimation)g(process,)f(w)o(as)i
(the)g(ne)o(xt)e(best)i(performed.)i(Their)1974 880 y(study)28
b(also)h(reported)e(prior)h(e)n(vidence)g(of)g(the)h(e)o(xtensi)n(v)o
(e)e(use)i(of)1974 980 y(e)o(xpert)19 b(estimation)g(in)i(industry)-5
b(.)23 b(The)d(results)g(of)g(a)g(recent)g(surv)o(e)o(y)1974
1079 y(of)25 b(softw)o(are)h(houses)f(has)h(pro)o(vided)d(further)h(v)o
(eri\002cation)h(of)g(the)1974 1179 y(widespread)g(use)h(of)g(e)o
(xpert)f(judgment)f(as)j(either)f(the)g(only)g(esti-)1974
1279 y(mation)j(method)f(used)h(or)h(as)g(a)g(back-up)d(to)j(other)f
(models)g([3)o(].)1974 1378 y([5)o(])f(pro)o(vide)e(further)g(e)n
(vidence)g(of)i(the)g(ef)n(fecti)n(v)o(eness)e(of)h(e)o(xpert)1974
1478 y(judgment)j(as)i(an)g(estimation)f(method)g(in)h(a)g(small)g(e)o
(xperiment)1974 1577 y(based)h(on)g(the)h(personal)e(softw)o(are)i
(process)f([7)o(].)65 b([6])33 b(also)h(re-)1974 1677
y(ports)17 b(the)i(results)f(of)g(a)g(case)h(study)e(of)h(estimation)f
(practices,)h(ulti-)1974 1777 y(mately)k(suggesting)f(that)h(research)f
(should)g(be)h(directed)g(to)n(w)o(ards)1974 1876 y Fk(supporting)i
Fj(e)o(xpert)h(judgment)f(rather)i(than)f(abandoning)e(it)k(as)g(a)1974
1976 y(viable)20 b(technique.)2073 2102 y(This)27 b(rather)f(pragmatic)
f(approach)g(may)h(be)g(e)n(v)o(en)g(more)g(fea-)1974
2202 y(sible)32 b(if)h(more)e(formally)g(combined)f(with)j(the)f
(\223estimation)g(by)1974 2301 y(analogy\224)19 b(method.)27
b(Experts)21 b(pro)o(vide)e(their)i(estimates)h(based)f(on)1974
2401 y(what)32 b(the)o(y)f(recall)h(of)g(pre)n(vious)f(similar)h(\(or)f
(analogous\))f(mod-)1974 2500 y(ules)21 b(or)f(projects,)h(perhaps)e
(with)i(adjustments)f(to)h(cope)f(with)h(an)o(y)1974
2600 y(dif)n(ferences)c(between)i(the)g(pre)n(viously)e(b)n(uilt)i
(objects)g(and)g(the)g(ob-)1974 2700 y(ject)27 b(to)g(be)g(de)n(v)o
(eloped.)42 b([4)o(])27 b(found)e(that)i(these)g(tw)o(o)g(approaches)
1974 2799 y(were)h(f)o(ar)f(more)g(widely)h(emplo)o(yed)e(in)i
(industry)e(than)i(algorith-)1974 2899 y(mic)18 b(models,)g(a)h
(\002nding)e(also)h(supported)f(by)g([8].)24 b(As)19
b(a)g(result,)f(the)1974 2999 y(analogy)d(approach)f(has)j(also)g(seen)
g(a)g(greater)f(de)o(gree)f(of)h(e)o(xposure)1974 3098
y(recently)j(\(e.g.)24 b(see)d([12)o(]\).)1974 3410 y
Fl(2)99 b(Misestimation)24 b(of)h(eff)n(ort)2073 3705
y Fj(Despite)h(the)f(importance)e(of)i(accurate)f(ef)n(fort)g
(estimates,)j(ac-)1974 3804 y(ceptable)18 b(le)n(v)o(els)i(of)f
(accurac)o(y)e(for)i(ef)n(fort)f(estimation)h(are)g(surpris-)1974
3904 y(ingly)j(lo)n(w)g(with)h(a)f(common)f(measure)h(for)f(a)i
Fk(good)h Fj(model)d(being)1974 4004 y(the)27 b(attainment)f(of)h
(estimates)h(that)f(are)g(within)g(25\045)f(of)h(the)g(ac-)1974
4103 y(tual)f(\002gure)g(at)h(least)g(75\045)f(of)g(the)g(time)h
(\(this)f(can)g(be)g(written)g(as)1974 4203 y Fg(pr)r(ed)p
Fh(\(0)p Fg(:)p Fh(25\))g Ff(\025)g Fh(0)p Fg(:)p Fh(75)p
Fj(\).)k(Whilst)23 b(this)g(le)n(v)o(el)f(of)f(accurac)o(y)g(may)g(not)
1974 4302 y(appear)30 b(ambitious)g(it)i(is)g(seldom)e(met)h(in)g
(practice,)i(or)e(e)n(v)o(en)f(in)1974 4402 y(post-hoc)21
b(academic)i(studies.)33 b(Thus)23 b(it)h(can)f(be)g(seen)g(that)g
(mises-)1974 4502 y(timation)f(is)i(more)e(the)i(norm)d(for)i(softw)o
(are)f(metric)h(models)g(than)1974 4601 y(an)d(occasional)f(and)h
(unfortunate)d(occurrence.)2073 4727 y(It)24 b(might)e(be)h(supposed)f
(that)h(the)g(misestimation)g(of)f(de)n(v)o(elop-)1974
4827 y(ment)g(ef)n(fort)f(required)f(for)i(modules)f(within)h(a)g
(system)h(\(and)e(also)1974 4926 y(for)29 b(systems)h(themselv)o(es)f
(within)h(an)f(or)o(ganization\))d(is)31 b(not)f(en-)1974
5026 y(tirely)d(random)e(when)h(these)h(estimates)g(include)f(subjecti)
n(v)o(e)g(hu-)1974 5126 y(man)d(input.)35 b(Some)23 b(associations,)h
(such)g(as)g(between)f(the)h(proba-)1974 5225 y(bility)f(of)g
(signi\002cant)g(misestimation)g(and)g(the)h(sizes)g(or)f(types)g(of)
1974 5325 y(the)d(modules/systems,)f(may)h(be)g(found)e(to)j(e)o(xist.)
1859 5574 y(2)p eop
%%Page: 3 3
3 2 bop -83 83 a Fj(Further)24 b(e)n(vidence)g(of)g(the)h(plausibility)
f(of)h(such)g(ef)n(fects)f(may)-182 183 y(be)c(tak)o(en)h(from)f(the)h
(follo)n(wing)e(comment)g(from)h(one)h(of)f(the)h(soft-)-182
282 y(w)o(are)39 b(houses)g(referred)f(to)i(abo)o(v)o(e:)62
b(\223W)-7 b(e)41 b(still)g(get)e(it)h(wrong.)-182 382
y(W)-7 b(e)30 b(under)e(estimate)h(constantly)-5 b(.)50
b(Not)30 b(so)f(much)f(on)h(the)g(small)-182 482 y(projects,)19
b(b)n(ut)h(quite)g(signi\002cantly)f(on)h(the)g(lar)o(ger)f(ones.)-6
b(\224)-83 583 y(These)37 b(associations)f(between)g(characteristics)g
(of)h(projects)-182 683 y(and)18 b(the)g(lik)o(elihood)f(of)i
(misestimation)f(may)g(e)o(xist)h(due)f(to)h(a)g(lar)o(ge)-182
782 y(number)f(of)i(causes)g(including:)-79 973 y(1.)41
b(changes)27 b(in)i(technology)d(that)i(are)h(not)f(fully)f(understood)
25 1072 y(in)e(terms)g(of)f(their)h(ef)n(fect)f(on)g(ef)n(fort\(such)e
(as)k(ne)n(wer)e(tools)25 1172 y(that)c(mak)o(es)g(some)g(types)h(of)e
(module)g(easier)i(to)f(de)n(v)o(elop\),)-79 1346 y(2.)41
b(le)n(v)o(els)17 b(of)f(personnel)f(e)o(xperience)g(and)h(skills)h
(\(such)f(f)o(actors)25 1445 y(are)26 b(often)g(dif)n(\002cult)f(to)i
(assess,)i(not)d(to)g(mention)f(personal)25 1545 y(biases)c(that)f
(project)f(managers)g(may)h(ha)n(v)o(e\),)-79 1719 y(3.)41
b(a)18 b(lack)g(of)g(understanding)d(of)i(the)h(module/system)e
(charac-)25 1818 y(teristics)26 b(\(for)d(e)o(xample,)h(some)g
(features)g(may)g(be)h(seen)f(as)25 1918 y(simple)j(by)f(a)h
(particular)e(project)h(manager)f(when)h(in)h(f)o(act)25
2017 y(the)o(y)20 b(require)e(substantial)i(de)n(v)o(elopment)e(ef)n
(fort\),)-79 2191 y(4.)41 b(other)16 b(in\003uences)g(due)g(to)h(the)g
(estimator')-5 b(s)17 b(background,)c(or)-79 2365 y(5.)41
b(an)o(y)21 b(number)e(of)i(other)g(related)f(reasons)h(\(including)e
(polit-)25 2465 y(ical)28 b(and)f(moti)n(v)n(ational)f(goals)i(\226)f
(the)h(\223price-to-win\224)d(ap-)25 2564 y(proach,)31
b(whereby)e(a)h(lo)n(w)g(estimate)h(is)g(pro)o(vided)c(simply)25
2664 y(in)e(order)e(to)h(win)h(a)g(contract)e(in)h(a)h(competiti)n(v)o
(e)e(tendering)25 2763 y(situation,)i(is)h(an)e(e)o(xample)g(of)g(ho)n
(w)g(these)h(goals)g(may)f(cir)n(-)25 2863 y(cumv)o(ent)19
b(the)h(estimation)f(process\).)-83 3053 y(In)c(short,)h(the)f(e)o
(xistence)g(of)g(such)g(consistent)f(biases)i(in)g(subjec-)-182
3153 y(ti)n(v)o(e)25 b(\(or)f(partially)h(subjecti)n(v)o(e\))f
(estimates)i(seems)g(plausible)e(and)-182 3253 y(could)h(e)n(v)o(en)f
(be)i(ar)o(gued)e(as)j(being)e(ine)n(vitable)g(with)h(an)o(y)f(estima-)
-182 3352 y(tion)20 b(procedure)d(in)m(v)n(olving)h(such)i(subjecti)n
(vity)-5 b(.)-83 3454 y(The)28 b(question)e(w)o(ould)h(seem)h(to)g(be)g
(less)h(that)f(of)f(whether)g(or)-182 3553 y(not)e(such)h(biases)h(e)o
(xist)f(and)g(more)f(concerned)f(with)j(the)f Fk(signi\002-)-182
3653 y(cance)16 b Fj(of)h(these)h(biases.)24 b(The)17
b(question)f(of)i(signi\002cance)e(is)i(harder)-182 3753
y(to)j(answer)f(since)h(it)g(depends)f(on)g(or)o(ganizational)d
(characteristics)-182 3852 y(\(such)29 b(as)h(a)n(v)n(ailable)f(e)o
(xcess)h(capability)-5 b(,)31 b(project)e(cost)h(mar)o(gins,)-182
3952 y(project)18 b(portfolio)g(construction,)f(and)i(so)g(on\),)g(and)
g(here)f(only)h(the)-182 4052 y(issue)k(of)f(identifying)e
(statistically)j(signi\002cant)f(associations)g(be-)-182
4151 y(tween)31 b(module)f(characteristics)i(and)f(estimation)g
(performance)-182 4251 y(will)21 b(be)f(pursued.)-182
4489 y Fl(3)99 b(Uses)20 b(of)h(models)g(of)f(systematic)h(biases)f(in)
h(eff)n(ort)-33 4605 y(estimation)-83 4827 y Fj(Kno)n(wing)g(the)g
(direction)g(and)g(magnitude)f(of)i(an)o(y)f(systematic)-182
4926 y(biasing)26 b(ef)n(fect)h(could)f(be)i(seen)f(as)h(signi\002cant)
f(to)g(project)g(man-)-182 5026 y(agement)j(for)h(at)i(least)f(three)g
(reasons.)59 b(First,)35 b(since)d(much)f(ef-)-182 5126
y(fort)18 b(estimation)g(is)h(performed)d(using)i(e)o(xpert)f(opinion)g
(or)i(subjec-)-182 5225 y(ti)n(v)o(ely)25 b(calibrated)h(models)f(then)
h(an)o(y)g(information)e(that)i(enables)-182 5325 y(bias)i(correction)f
(in)h(these)h(estimates)g(w)o(ould)f(be)g(in)m(v)n(aluable)f(for)1974
83 y(project)k(management)f(purposes.)58 b(Errors)31
b(that)h(are)g(systemati-)1974 183 y(cally)26 b(made)g(may)g(be)g(pre)n
(v)o(entable)e(by)i(educating)f(project)g(man-)1974 282
y(agers)f(that)h(the)o(y)f(are)h(making)e(such)h(errors)g(or)g(by)h
(automatically)1974 382 y(correcting)14 b(estimates)j(in)g(such)f
(circumstances.)23 b(A)17 b(manager)e(who)1974 482 y
Fk(consistently)21 b Fj(underestimates)g(the)h(ef)n(fort)f(required)f
(for)i(database)1974 581 y(modules,)27 b(and)g(underestimates)e(the)i
(ef)n(fort)f(for)h(lar)o(ger)e(modules)1974 681 y(is)19
b(just)f(as)h(v)n(aluable)e(in)h(project)f(management)f(as)j(an)f
(equally)f(con-)1974 780 y(sistent)g(correct)f(estimator)g(pro)o
(viding)e(that)j(the)g(necessary)f(adjust-)1974 880 y(ments)i(are)g
(kno)n(wn.)23 b(In)18 b(some)f(sense)i(such)f(adjustments)f(may)h(also)
1974 980 y(be)27 b(technology)f(independent,)g(or)i(at)g(least)g(more)f
(so)h(than)f(algo-)1974 1079 y(rithmic)18 b(estimating)h(models)f
(themselv)o(es.)24 b(This)19 b(w)o(ould)g(possibly)1974
1179 y(enable)27 b(their)g(use)h(for)f(corrections)f(on)i(ne)n(w)f
(forms)g(of)g(de)n(v)o(elop-)1974 1279 y(ment)20 b(where)f(less)j
(empirical)d(data)h(is)h(a)n(v)n(ailable.)2073 1379 y(Second,)h(an)o(y)
g(such)g(associations)h(w)o(ould)f(indicate)g(that)g(mod-)1974
1479 y(els)d(that)g(treat)g(the)f(dif)n(ference)f(between)h(estimates)h
(and)f(actual)g(ef-)1974 1578 y(fort)h(as)h(independent)d(and)j
(identically)e(distrib)n(uted)h(random)f(v)n(ari-)1974
1678 y(ables)i(w)o(ould)g(be)g(\003a)o(wed.)2073 1779
y(The)34 b(third)f(reason)h(for)f(such)h(models)f(being)g(useful)g(is)i
(that)1974 1878 y(kno)n(wing)15 b(which)i(types)g(of)f(modules)g(or)h
(systems)h(are)f(more)f(lik)o(ely)1974 1978 y(to)31 b(be)h
(misestimated)f(allo)n(ws)g(project)g(managers)f(to)h(assess)i(the)1974
2078 y(risk)28 b(of)g(estimates.)48 b(F)o(or)28 b(e)o(xample,)g(for)f
(modules)g(that)h(are)g(high)1974 2177 y(risk)i(for)g(underestimating)e
(it)j(may)f(be)g(necessary)g(to)g(allo)n(w)h(for)1974
2277 y(more)18 b(slack)i(in)f(the)h(process.)k(There)18
b(has)i(been)e(little)j(w)o(ork)d(on)h(the)1974 2377
y(risk)25 b(of)g(softw)o(are)f(metric)h(models)g(and)f(this)i(could)e
(be)h(seen)g(as)h(a)1974 2476 y(useful)20 b(foundation)d(for)j(such)f
(w)o(ork)h(to)g(be)g(b)n(uilt)h(on.)1974 2712 y Fl(4)99
b(Health)25 b(pr)n(oject)h(data)2073 2931 y Fj(In)31
b(order)f(to)h(e)o(xamine)f(the)h(possibility)f(and)h(nature)f(of)g
(such)1974 3030 y(misestimation)j(patterns)g(a)h(data)g(set)h(of)e(77)g
(observ)n(ations)f(w)o(as)1974 3130 y(formed)c(from)g(a)i(lar)o(ge)f
(health)g(system)g(data)h(set.)53 b(The)29 b(original)1974
3230 y(data)h(set)i(contained)d(85)h(modules)g(for)g(which)g(all)h
(module)e(data)1974 3329 y(w)o(as)k(a)n(v)n(ailable,)h(less)f(three)e
(without)g(ef)n(fort)g(estimates)h(and)g(an-)1974 3429
y(other)19 b(\002)n(v)o(e)h(which)f(included)g(both)g(reports)g(and)g
(screens)h(\(without)1974 3529 y(pro)o(viding)g(separate)i(ef)n(fort)g
(measures)g(or)h(estimates\).)32 b(The)23 b(v)n(ari-)1974
3628 y(ables)29 b(and)f(their)g(associated)g(codings)g(sho)n(wn)f(in)i
(T)-7 b(able)29 b(1)f(were)1974 3728 y(used.)2073 3828
y(As)20 b(can)e(be)h(seen)f(the)h Fk(Modi\002es)p Fj(,)f
Fk(T)-6 b(ype)p Fj(,)18 b Fk(Entry)p Fj(,)h(and)f Fk(Links)h
Fj(v)n(ari-)1974 3928 y(ables)25 b(are)g(all)g(binary)-5
b(,)24 b(and)g(there)h(are)g(three)f(le)n(v)o(els)h(for)f
Fk(Estimate)1974 4028 y Fj(and)18 b Fk(Size)p Fj(.)25
b(If)18 b Fk(Estimate)h Fj(is)g(treated)f(as)i(the)e(response)g(v)n
(ariable)g(then)1974 4127 y(there)i(are)g(48)f(le)n(v)o(els)i(in)f(the)
g(independent)e(v)n(ariables.)2073 4228 y(This)31 b(data)g(can)f(be)g
(analyzed)f(in)i(terms)g(of)f(counts)g(for)g(each)1974
4328 y(le)n(v)o(el)20 b(of)g(each)g(v)n(ariable.)k(Since)d(there)f(are)
g(six)h(v)n(ariables)e(it)i(is)h(dif-)1974 4427 y(\002cult)e(to)h
(meaningfully)c(illustrate)j(the)g(frequencies)f(so)h(that)h(rela-)1974
4527 y(tionships)g(can)h(be)h(noted)e(for)g(further)g(in)m(v)o
(estigation,)f(and)i(also)g(to)1974 4627 y(check)d(that)h(the)h(cells)g
(ha)n(v)o(e)e(a)i(reasonable)e(count.)2073 4727 y(T)-7
b(ables)21 b(of)f(counts)f(for)g(each)h(v)n(ariable)f(can)h(be)g
(constructed)e(in-)1974 4827 y(di)n(vidually)-5 b(.)49
b(All)30 b(v)n(ariables)e(sho)n(w)h(a)h(reasonable)d(split)j(between)
1974 4926 y(classes,)36 b(with)c(only)g Fk(Entry)g Fj(class)h(0)g(ha)n
(ving)e(a)i(proportion)c(less)1974 5026 y(than)c(25\045,)i(in)g(this)f
(case)h(13\045.)42 b(All)27 b(other)e(classes)i(for)e(all)i(v)n(ari-)
1974 5126 y(ables)22 b(contain)f(at)h(least)h(25\045)f(of)f(the)h
(observ)n(ations)f(for)g(that)h(v)n(ari-)1974 5225 y(able,)i(which)f
(seems)h(adequate)f(for)g(the)g(purposes)g(of)g(this)h(analy-)1974
5325 y(sis.)1859 5574 y(3)p eop
%%Page: 4 4
4 3 bop 314 3 3132 4 v 312 103 4 100 v 364 73 a Fj(V)-9
b(ariable)p 708 103 V 117 w(Code)p 982 103 V 99 w(Description)p
3444 103 V 314 106 3132 4 v 312 206 4 100 v 364 176 a(Modi\002es)p
708 206 V 99 w(0)p 982 206 V 233 w(The)20 b(sum)g(of)g(create,)f
(modify)-5 b(,)18 b(and)i(delete)g(transactions)f(is)j(zero.)p
3444 206 V 312 306 V 708 306 V 759 276 a(1)p 982 306
V 233 w(The)e(sum)g(of)g(data-altering)e(transactions)h(is)j(non-zero.)
p 3444 306 V 314 309 3132 4 v 312 408 4 100 v 364 379
a(T)-7 b(ype)p 708 408 V 230 w(0)p 982 408 V 233 w(The)20
b(module)f(is)i(made)e(up)h(entirely)f(of)h(screens.)p
3444 408 V 312 508 V 708 508 V 759 478 a(1)p 982 508
V 233 w(The)g(module)f(is)i(made)e(up)h(entirely)f(of)h(reports.)p
3444 508 V 314 511 3132 4 v 312 711 4 200 v 364 581 a(Entry)p
708 711 V 209 w(0)p 982 711 V 233 w(Data)j(cannot)f(be)h(entered)f
(from)f(this)j(module)d(\(for)h(e)o(xample,)g(report)g(parame-)1034
681 y(ters)f(or)e(search)h(criteria)g(as)h(well)g(as)g(standard)e(data)
h(entry)f(and)h(editing\).)p 3444 711 V 312 810 4 100
v 708 810 V 759 780 a(1)p 982 810 V 233 w(Data)g(can)g(be)g(entered)g
(by)f(the)i(user)-5 b(.)p 3444 810 V 314 814 3132 4 v
312 913 4 100 v 364 883 a(Links)p 708 913 V 205 w(0)p
982 913 V 233 w(Only)20 b(one)f(table)h(w)o(as)h(used)f(\(all)h
(modules)e(accessed)h(at)h(least)g(one)e(table\).)p 3444
913 V 312 1013 V 708 1013 V 759 983 a(1)p 982 1013 V
233 w(More)g(than)h(one)g(table)g(w)o(as)h(accessed.)p
3444 1013 V 314 1016 3132 4 v 312 1116 4 100 v 364 1086
a(Estimate)p 708 1116 V 104 w(-1)p 982 1116 V 205 w(The)f(ef)n(fort)f
(w)o(as)i(at)g(least)f(35\045)g(underestimated.)p 3444
1116 V 312 1215 V 708 1215 V 759 1186 a(0)p 982 1215
V 233 w(The)g(estimated)g(ef)n(fort)f(w)o(as)i(within)f(35\045)g(of)g
(the)g(actual.)p 3444 1215 V 312 1315 V 708 1315 V 759
1285 a(1)p 982 1315 V 233 w(The)g(ef)n(fort)f(w)o(as)i(at)g(least)f
(35\045)g(o)o(v)o(erestimated.)p 3444 1315 V 314 1318
3132 4 v 312 1418 4 100 v 364 1388 a(Size)p 708 1418
V 252 w(1)p 982 1418 V 233 w(The)g(bottom)f(one-third)f(of)i(the)g
(modules)f(as)i(sized)f(by)g(actual)g(size.)p 3444 1418
V 312 1518 V 708 1518 V 759 1488 a(2)p 982 1518 V 233
w(The)g(middle)f(one-third)f(of)i(the)g(modules)f(as)i(sized)g(by)f
(actual)g(size.)p 3444 1518 V 312 1617 V 708 1617 V 759
1587 a(3)p 982 1617 V 233 w(The)g(top)g(one-third)e(of)i(the)g(modules)
f(as)i(sized)f(by)g(actual)g(size.)p 3444 1617 V 314
1621 3132 4 v 1198 1771 a Fe(T)-7 b(ab)o(le)23 b(1.)g(The)g(v)n(ariab)o
(les)i(in)e(the)f(model)p 547 1908 2666 4 v 545 2008
4 100 v 931 2008 V 1248 1978 a Fk(Entry)p 1480 2008 V
410 w(Links)p 2071 2008 V 305 w(Modi\002es)p 2662 2008
V 414 w(Size)p 3211 2008 V 547 2011 2666 4 v 545 2111
4 100 v 699 2081 a(Entry)p 931 2111 V 521 w Fj(-)p 1480
2111 V 562 w(-)p 2071 2111 V 563 w(-)p 2662 2111 V 521
w(-)p 3211 2111 V 547 2114 2666 4 v 545 2214 4 100 v
703 2184 a Fk(Links)p 931 2214 V 100 w Fj(3.765)c(\(0.052\))p
1480 2214 V 560 w(-)p 2071 2214 V 563 w(-)p 2662 2214
V 521 w(-)p 3211 2214 V 547 2217 2666 4 v 545 2317 4
100 v 597 2287 a Fk(Modi\002es)p 931 2317 V 99 w Fj(0.011)g(\(0.915\))p
1480 2317 V 180 w(6.715)g(\(0.10\))p 2071 2317 V 561
w(-)p 2662 2317 V 521 w(-)p 3211 2317 V 547 2320 2666
4 v 545 2420 4 100 v 749 2390 a Fk(Size)p 931 2420 V
100 w Fj(1.495)g(\(0.473\))p 1480 2420 V 97 w(49.479)g(\(0.000\))p
2071 2420 V 138 w(4.412)h(\(0.110\))p 2662 2420 V 518
w(-)p 3211 2420 V 547 2423 2666 4 v 545 2522 4 100 v
728 2493 a Fk(T)-6 b(ype)p 931 2522 V 99 w Fj(0.135)18
b(\(0.713\))p 1480 2522 V 138 w(8.263)h(\(0.004\))p 2071
2522 V 97 w(54.110)f(\(0.000\))p 2662 2522 V 96 w(7.852)h(\(0.020\))p
3211 2522 V 547 2526 2666 4 v 826 2676 a Fe(T)-7 b(ab)o(le)24
b(2.)f(Associations)g(between)f(the)h(predictor)g(v)n(ariab)o(les)-83
3025 y Fj(The)41 b(data)f(w)o(as)i(then)e(analyzed)g(using)g
(contingenc)o(y)e(table)-182 3125 y(analysis,)20 b(logistic)h(re)o
(gression,)d(and)i(log-linear)f(modeling.)24 b(Con-)-182
3225 y(tingenc)o(y)h(analysis)i(is)h(used)f(to)h(initially)f(determine)
f(the)h(associ-)-182 3324 y(ations)k(between)g(the)h
Fk(Estimate)g Fj(type)f(and)g(the)h(other)f(\002)n(v)o(e)h(v)n(ari-)
-182 3424 y(ables.)24 b(Logistic)18 b(re)o(gression)g(is)h(used)f(to)h
(determine)e(the)i(ability)f(of)-182 3523 y(the)25 b(independent)e(v)n
(ariables)h(to)h(predict)g(underestimates,)g(errors)-182
3623 y(in)i(estimation,)g(and)g(o)o(v)o(erestimates)e(\(using)h(the)h
(35\045)f(threshold)-182 3723 y(in)21 b(each)g(case\).)27
b(Log-linear)19 b(modeling)h(is)i(then)e(used)h(to)g(e)o(xamine)-182
3822 y(the)29 b(main)f(predictor)g(v)n(ariables)g(in)h(terms)g(of)g
(associations)g(with)-182 3922 y(the)20 b(three)g(le)n(v)o(els)g(of)g
Fk(Estimate)p Fj(.)-182 4138 y Fl(5)99 b(Contingency)26
b(table)f(analysis)-182 4337 y Fd(5.1)90 b(Intr)n(oduction)-83
4528 y Fj(Contingenc)o(y)20 b(table)i(analysis)h(is)g(suited)f(to)g
(analyzing)f(the)h(in-)-182 4628 y(dependence)27 b(or)i(otherwise)g(of)
h(the)f(relationships)g(between)g(the)-182 4727 y(v)n(ariables)24
b(in)i(terms)g(of)f(the)h(counts)f(in)h(each)f(cell.)41
b(If)26 b(the)g(counts)-182 4827 y(follo)n(w)20 b(the)h(mar)o(ginal)f
(e)o(xpectations)f(then)i(independence)d(can)j(be)-182
4926 y(concluded.)40 b(Otherwise,)27 b(an)f(association)f(e)o(xists)h
(and)g(will)h(need)-182 5026 y(to)20 b(be)g(further)f(in)m(v)o
(estigated.)1974 3025 y Fd(5.2)90 b(Analysis)2073 3233
y Fj(There)25 b(are)g(associations)f(among)g(the)h(predictor)e(v)n
(ariables)h(as)1974 3332 y(sho)n(wn)i(in)i(T)-7 b(able)27
b(2)h(where)e(the)i Fg(\037)3001 3302 y Fc(2)3066 3332
y Fj(statistics)h(and)d(their)h(signi\002-)1974 3432
y(cance)g(le)n(v)o(els)g(are)g(reported.)44 b(The)27
b(signi\002cant)f(associations)h(are)1974 3532 y(between)18
b Fk(Size)i Fj(and)f Fk(Links)p Fj(,)h Fk(T)-6 b(ype)19
b Fj(and)g Fk(Links)p Fj(,)h Fk(T)-6 b(ype)19 b Fj(and)g
Fk(Modi\002es)p Fj(,)1974 3631 y(and)g Fk(T)-6 b(ype)20
b Fj(and)g Fk(Size)p Fj(.)2073 3731 y(When)h(analyzing)e(the)i(data)g
(using)f Fg(\037)3173 3701 y Fc(2)3210 3731 y Fj(,)i(Phi,)f(Cramer')-5
b(s)20 b(V)-11 b(,)22 b(and)1974 3831 y(the)17 b(Contingenc)o(y)d(Coef)
n(\002cient)j(to)g(test)h(for)e(associations)h(between)1974
3930 y(each)24 b(v)n(ariable)e(\(e)o(xcept)h Fk(Estimate)p
Fj(\))h(and)f Fk(Estimate)i Fj(the)f(results)g(in)1974
4030 y(T)-7 b(able)27 b(3)h(were)f(found.)45 b(The)27
b(crosstabs)g(are)g(sho)n(wn)g(in)g(T)-7 b(ables)28 b(4)1974
4129 y(to)20 b(8.)2073 4229 y(From)h(T)-7 b(able)21 b(3)g(it)h(can)f
(be)g(seen)g(that)g(the)g(v)n(alue)f(of)h Fk(Estimate)g
Fj(is)1974 4329 y(associated)d(with)h Fk(Size)p Fj(,)g
Fk(T)-6 b(ype)p Fj(,)18 b(and)g Fk(Links)p Fj(.)25 b(There)17
b(is)j(also)f(a)f(poten-)1974 4428 y(tial)27 b(relationship)e(with)h
Fk(Modi\002es)p Fj(.)43 b Fk(Entry)27 b Fj(does)f(not)g(seem)h(to)f(be)
1974 4528 y(associated)d(with)h(the)g(de)n(v)o(elopment)c(ef)n(fort)j
(estimate.)35 b(Ho)n(we)n(v)o(er)m(,)1974 4628 y(it)20
b(should)f(be)h(remembered)d(that)j(these)g(v)n(ariables)f(are)h
(intercorre-)1974 4727 y(lated,)e(with,)h(for)e(e)o(xample,)g(smaller)h
(modules)g(being)f(more)g(lik)o(ely)1974 4827 y(to)j(contain)e(no)h
(table)h(links.)25 b(The)19 b(f)o(act)h(that)f(such)h(correlations)e(e)
o(x-)1974 4926 y(ist)j(suggests)g(that)g(not)f(all)h(v)n(ariables)f
(may)g(be)g(needed)f(in)i(a)g(model)1974 5026 y(for)e
Fk(Estimate)p Fj(.)1859 5574 y(4)p eop
%%Page: 5 5
5 4 bop -144 124 1892 4 v -146 224 4 100 v -94 194 a
Fj(V)-9 b(ariable)p 240 224 V 107 w(Statistic)p 1132
224 V 665 w(V)g(alue)p 1460 224 V 167 w(sig.)p 1746 224
V -144 227 1892 4 v -146 327 4 100 v -94 297 a Fk(Size)p
240 327 V 251 w Fg(\037)343 267 y Fc(2)p 1132 327 V 1183
297 a Fj(37.079)p 1460 327 V 97 w(0.000)p 1746 327 V
-146 426 V 240 426 V 291 396 a(Lik)o(elihood)19 b(Ratio)p
1132 426 V 327 w(41.004)p 1460 426 V 97 w(0.000)p 1746
426 V -146 526 V 240 526 V 291 496 a(Phi)p 1132 526 V
823 w(0.694)p 1460 526 V 97 w(0.000)p 1746 526 V -146
626 V 240 626 V 291 596 a(Cramer')-5 b(s)21 b(V)p 1132
626 V 548 w(0.491)p 1460 626 V 97 w(0.000)p 1746 626
V -146 725 V 240 725 V 291 695 a(Contingenc)o(y)d(coef)n(\002cient)p
1132 725 V 140 w(0.570)p 1460 725 V 97 w(0.000)p 1746
725 V -144 729 1892 4 v -146 828 4 100 v -94 798 a Fk(T)-6
b(ype)p 240 828 V 229 w Fg(\037)343 768 y Fc(2)p 1132
828 V 1225 798 a Fj(8.886)p 1460 828 V 97 w(0.012)p 1746
828 V -146 928 V 240 928 V 291 898 a(Lik)o(elihood)19
b(Ratio)p 1132 928 V 327 w(10.874)p 1460 928 V 97 w(0.004)p
1746 928 V -146 1027 V 240 1027 V 291 998 a(Phi)p 1132
1027 V 823 w(0.340)p 1460 1027 V 97 w(0.012)p 1746 1027
V -146 1127 V 240 1127 V 291 1097 a(Cramer')-5 b(s)21
b(V)p 1132 1127 V 548 w(0.340)p 1460 1127 V 97 w(0.012)p
1746 1127 V -146 1227 V 240 1227 V 291 1197 a(Contingenc)o(y)d(coef)n
(\002cient)p 1132 1227 V 140 w(0.322)p 1460 1227 V 97
w(0.012)p 1746 1227 V -144 1230 1892 4 v -146 1330 4
100 v -94 1300 a Fk(Entry)p 240 1330 V 200 w Fg(\037)343
1270 y Fc(2)p 1132 1330 V 1225 1300 a Fj(2.854)p 1460
1330 V 97 w(0.240)p 1746 1330 V -146 1429 V 240 1429
V 291 1399 a(Lik)o(elihood)h(Ratio)p 1132 1429 V 369
w(3.315)p 1460 1429 V 97 w(0.191)p 1746 1429 V -146 1529
V 240 1529 V 291 1499 a(Phi)p 1132 1529 V 823 w(0.193)p
1460 1529 V 97 w(0.240)p 1746 1529 V -146 1628 V 240
1628 V 291 1599 a(Cramer')-5 b(s)21 b(V)p 1132 1628 V
548 w(0.193)p 1460 1628 V 97 w(0.240)p 1746 1628 V -146
1728 V 240 1728 V 291 1698 a(Contingenc)o(y)d(coef)n(\002cient)p
1132 1728 V 140 w(0.189)p 1460 1728 V 97 w(0.240)p 1746
1728 V -144 1731 1892 4 v -146 1831 4 100 v -94 1801
a Fk(Links)p 240 1831 V 205 w Fg(\037)343 1771 y Fc(2)p
1132 1831 V 1183 1801 a Fj(38.047)p 1460 1831 V 97 w(0.000)p
1746 1831 V -146 1931 V 240 1931 V 291 1901 a(Lik)o(elihood)h(Ratio)p
1132 1931 V 327 w(41.699)p 1460 1931 V 97 w(0.000)p 1746
1931 V -146 2030 V 240 2030 V 291 2000 a(Phi)p 1132 2030
V 823 w(0.703)p 1460 2030 V 97 w(0.000)p 1746 2030 V
-146 2130 V 240 2130 V 291 2100 a(Cramer')-5 b(s)21 b(V)p
1132 2130 V 548 w(0.703)p 1460 2130 V 97 w(0.000)p 1746
2130 V -146 2230 V 240 2230 V 291 2200 a(Contingenc)o(y)d(coef)n
(\002cient)p 1132 2230 V 140 w(0.575)p 1460 2230 V 97
w(0.000)p 1746 2230 V -144 2233 1892 4 v -146 2332 4
100 v -94 2303 a Fk(Modi\002es)p 240 2332 V 98 w Fg(\037)343
2272 y Fc(2)p 1132 2332 V 1225 2303 a Fj(5.019)p 1460
2332 V 97 w(0.081)p 1746 2332 V -146 2432 V 240 2432
V 291 2402 a(Lik)o(elihood)h(Ratio)p 1132 2432 V 369
w(5.549)p 1460 2432 V 97 w(0.062)p 1746 2432 V -146 2532
V 240 2532 V 291 2502 a(Phi)p 1132 2532 V 823 w(0.255)p
1460 2532 V 97 w(0.081)p 1746 2532 V -146 2631 V 240
2631 V 291 2601 a(Cramer')-5 b(s)21 b(V)p 1132 2631 V
548 w(0.255)p 1460 2631 V 97 w(0.081)p 1746 2631 V -146
2731 V 240 2731 V 291 2701 a(Contingenc)o(y)d(coef)n(\002cient)p
1132 2731 V 140 w(0.247)p 1460 2731 V 97 w(0.081)p 1746
2731 V -144 2734 1892 4 v 130 2884 a Fe(T)-7 b(ab)o(le)23
b(3.)h(Associations)e(with)h Fk(Estimate)p 260 3280 1085
4 v 258 3380 4 100 v 526 3380 V 657 3350 a(Estimate)p
1343 3380 V 260 3383 1085 4 v 258 3483 4 100 v 310 3453
a(Size)p 526 3483 V 148 w Fj(-1)p 709 3483 V 140 w(0)p
892 3483 V 141 w(1)p 1074 3483 V 99 w(T)-7 b(otal)p 1343
3483 V 260 3486 1085 4 v 258 3586 4 100 v 310 3556 a(1)p
526 3586 V 267 w(1)p 709 3586 V 141 w(6)p 892 3586 V
99 w(19)p 1074 3586 V 184 w(26)p 1343 3586 V 258 3685
V 310 3656 a(2)p 526 3685 V 226 w(12)p 709 3685 V 140
w(9)p 892 3685 V 141 w(4)p 1074 3685 V 184 w(25)p 1343
3685 V 258 3785 V 310 3755 a(3)p 526 3785 V 226 w(19)p
709 3785 V 140 w(5)p 892 3785 V 141 w(2)p 1074 3785 V
184 w(26)p 1343 3785 V 260 3788 1085 4 v 258 3888 4 100
v 310 3858 a(T)g(otal)p 526 3888 V 99 w(32)p 709 3888
V 98 w(20)p 892 3888 V 99 w(25)p 1074 3888 V 184 w(77)p
1343 3888 V 260 3891 1085 4 v 331 4041 a Fe(T)g(ab)o(le)23
b(4.)g Fk(Size)g Fe(and)g Fk(Estimate)p 260 4443 V 258
4542 4 100 v 526 4542 V 657 4512 a(Estimate)p 1343 4542
V 260 4545 1085 4 v 258 4645 4 100 v 310 4615 a(T)-6
b(ype)p 526 4645 V 126 w Fj(-1)p 709 4645 V 140 w(0)p
892 4645 V 141 w(1)p 1074 4645 V 99 w(T)f(otal)p 1343
4645 V 260 4648 1085 4 v 258 4748 4 100 v 310 4718 a(1)p
526 4748 V 226 w(20)p 709 4748 V 98 w(14)p 892 4748 V
99 w(24)p 1074 4748 V 184 w(58)p 1343 4748 V 258 4848
V 310 4818 a(2)p 526 4848 V 226 w(12)p 709 4848 V 140
w(6)p 892 4848 V 141 w(1)p 1074 4848 V 184 w(19)p 1343
4848 V 260 4851 1085 4 v 258 4951 4 100 v 310 4921 a(T)g(otal)p
526 4951 V 99 w(32)p 709 4951 V 98 w(20)p 892 4951 V
99 w(25)p 1074 4951 V 184 w(77)p 1343 4951 V 260 4954
1085 4 v 320 5104 a Fe(T)g(ab)o(le)23 b(5.)h Fk(T)-6
b(ype)22 b Fe(and)h Fk(Estimate)p 2408 279 1101 4 v 2406
378 4 100 v 2691 378 V 2821 348 a(Estimate)p 3507 378
V 2408 382 1101 4 v 2406 481 4 100 v 2458 451 a(Entry)p
2691 481 V 113 w Fj(-1)p 2873 481 V 140 w(0)p 3056 481
V 141 w(1)p 3238 481 V 99 w(T)-7 b(otal)p 3507 481 V
2408 485 1101 4 v 2406 584 4 100 v 2458 554 a(0)p 2691
584 V 284 w(5)p 2873 584 V 140 w(4)p 3056 584 V 141 w(1)p
3238 584 V 185 w(10)p 3507 584 V 2406 684 V 2458 654
a(1)p 2691 684 V 242 w(27)p 2873 684 V 99 w(16)p 3056
684 V 98 w(24)p 3238 684 V 185 w(67)p 3507 684 V 2408
687 1101 4 v 2406 787 4 100 v 2458 757 a(T)g(otal)p 2691
787 V 115 w(32)p 2873 787 V 99 w(20)p 3056 787 V 98 w(25)p
3238 787 V 185 w(77)p 3507 787 V 2408 790 1101 4 v 2461
940 a Fe(T)g(ab)o(le)24 b(6.)f Fk(Entry)g Fe(and)f Fk(Estimate)p
2410 1661 1096 4 v 2408 1761 4 100 v 2688 1761 V 2819
1731 a(Estimate)p 3504 1761 V 2410 1764 1096 4 v 2408
1864 4 100 v 2460 1834 a(Links)p 2688 1864 V 114 w Fj(-1)p
2871 1864 V 140 w(0)p 3054 1864 V 141 w(1)p 3236 1864
V 99 w(T)-7 b(otal)p 3504 1864 V 2410 1867 1096 4 v 2408
1967 4 100 v 2460 1937 a(0)p 2688 1967 V 279 w(0)p 2871
1967 V 141 w(2)p 3054 1967 V 99 w(17)p 3236 1967 V 184
w(19)p 3504 1967 V 2408 2066 V 2460 2036 a(1)p 2688 2066
V 238 w(32)p 2871 2066 V 98 w(18)p 3054 2066 V 141 w(8)p
3236 2066 V 184 w(58)p 3504 2066 V 2410 2070 1096 4 v
2408 2169 4 100 v 2460 2139 a(T)g(otal)p 2688 2169 V
111 w(32)p 2871 2169 V 98 w(20)p 3054 2169 V 99 w(25)p
3236 2169 V 184 w(77)p 3504 2169 V 2410 2172 1096 4 v
2464 2323 a Fe(T)g(ab)o(le)23 b(7.)g Fk(Links)g Fe(and)g
Fk(Estimate)p 2357 3027 1202 4 v 2355 3127 4 100 v 2741
3127 V 2872 3097 a(Estimate)p 3557 3127 V 2357 3130 1202
4 v 2355 3230 4 100 v 2407 3200 a(Modi\002es)p 2741 3230
V 113 w Fj(-1)p 2924 3230 V 140 w(0)p 3107 3230 V 141
w(1)p 3289 3230 V 99 w(T)-7 b(otal)p 3557 3230 V 2357
3233 1202 4 v 2355 3333 4 100 v 2407 3303 a(0)p 2741
3333 V 344 w(12)p 2924 3333 V 140 w(7)p 3107 3333 V 141
w(3)p 3289 3333 V 184 w(22)p 3557 3333 V 2355 3433 V
2407 3403 a(1)p 2741 3433 V 344 w(20)p 2924 3433 V 98
w(13)p 3107 3433 V 99 w(22)p 3289 3433 V 184 w(55)p 3557
3433 V 2357 3436 1202 4 v 2355 3536 4 100 v 2407 3506
a(T)g(otal)p 2741 3536 V 217 w(32)p 2924 3536 V 98 w(20)p
3107 3536 V 99 w(25)p 3289 3536 V 184 w(77)p 3557 3536
V 2357 3539 1202 4 v 2411 3689 a Fe(T)g(ab)o(le)23 b(8.)g
Fk(Modi\002es)f Fe(and)h Fk(Estimate)p 2195 4410 1527
4 v 2193 4509 4 100 v 2245 4479 a Fj(Estimate)p 2584
4509 V 99 w(Model)d(1)p 2962 4509 V 99 w(Model)f(2)p
3341 4509 V 99 w(Model)h(3)p 3720 4509 V 2195 4513 1527
4 v 2193 4612 4 100 v 2245 4582 a(-1)p 2584 4612 V 320
w(1)p 2962 4612 V 337 w(1)p 3341 4612 V 336 w(0)p 3720
4612 V 2193 4712 V 2245 4682 a(0)p 2584 4712 V 348 w(0)p
2962 4712 V 337 w(0)p 3341 4712 V 336 w(0)p 3720 4712
V 2193 4812 V 2245 4782 a(1)p 2584 4812 V 348 w(0)p 2962
4812 V 337 w(1)p 3341 4812 V 336 w(1)p 3720 4812 V 2195
4815 1527 4 v 2452 4965 a Fe(T)-7 b(ab)o(le)24 b(9.)f(The)g(three)g
(models)1859 5574 y Fj(5)p eop
%%Page: 6 6
6 5 bop -182 83 a Fd(5.3)90 b(Conclusions)21 b(fr)n(om)i(contingency)d
(table)j(analysis)-83 289 y Fj(F)o(or)28 b Fk(Size)h
Fj(lar)o(ger)d(modules)i(are)g(more)f(lik)o(ely)h(to)g(be)h(underes-)
-182 389 y(timated)23 b(in)g(terms)h(of)f(ef)n(fort,)f(while)i(smaller)
f(modules)f(are)i(more)-182 489 y(lik)o(ely)h(to)g(be)g(o)o(v)o
(erestimated.)38 b(The)25 b(latter)h(aspect)f(is)h(predictable)-182
588 y(since)19 b(small)g(modules)f(will)h(generally)f(not)g(tak)o(e)h
(less)h(than)e(a)i(min-)-182 688 y(imal)e(amount)g(of)g(time)h(that)g
(re\003ects)g(the)f(general)g(o)o(v)o(erhead)e(asso-)-182
787 y(ciated)26 b(with)g(de)n(v)o(eloping)e(a)i(module)f(irrespecti)n
(v)o(e)g(of)h(its)h(size)g(or)-182 887 y(other)21 b(characteristics.)30
b(Ho)n(we)n(v)o(er)20 b(the)i(tendenc)o(y)f(for)g(lar)o(ge)g(mod-)-182
987 y(ules)28 b(to)g(be)h(underestimated)d(is)j(more)e(pronounced)e
(than)j(might)-182 1086 y(ha)n(v)o(e)19 b(been)h(e)o(xpected.)-83
1186 y(The)k Fk(T)-6 b(ype)23 b Fj(of)h(the)g(module)e(seems)i(to)g(af)
n(fect)g(ef)n(fort)e(estimates)-182 1286 y(in)e(the)g(manner)e(that)i
(screens)g(are)g(much)f(more)g(lik)o(ely)h(to)g(be)g(o)o(v)o(er)n(-)
-182 1385 y(estimated,)25 b(whereas)f(reports)g(are)h(much)f(more)g
(lik)o(ely)g(to)h(be)g(un-)-182 1485 y(derestimated.)d(This)17
b(indicates)f(a)h(de\002nite)f(bias)g(on)g(the)h(part)f(of)g(the)-182
1584 y(estimator)24 b(which)g(w)o(ould)g(be)g(interesting)g(to)h(in)m
(v)o(estigate)d(further)-182 1684 y(on)k(subsequent)f(projects)h(to)h
(see)g(if)g(it)h(is)f(a)g(general)f(bias)h(or)f(one)-182
1784 y(speci\002c)20 b(to)g(this)h(project.)-83 1883
y(Modules)e(without)g(an)o(y)g Fk(Links)h Fj(are)f(more)g(lik)o(ely)g
(to)h(be)g(o)o(v)o(eres-)-182 1983 y(timated,)e(while)h(those)f(with)h
Fk(Links)h Fj(are)e(more)g(lik)o(ely)h(to)g(be)f(under)n(-)-182
2083 y(estimated.)25 b(This)c(is)h(consistent)e(with)g(the)h
Fk(Size)g Fj(ef)n(fect)f(mentioned)-182 2182 y(abo)o(v)o(e)i(since)i
(lar)o(ger)e(modules)h(are)g(more)g(lik)o(ely)h(to)f(ha)n(v)o(e)h(se)n
(v)o(eral)-182 2282 y(tables)e(in)m(v)n(olv)o(ed.)27
b(While)c(this)f(v)n(ariable)f(may)g(be)h(interesting,)f(the)-182
2381 y(correlation)f(between)i Fk(Size)h Fj(and)f(itself)h(may)f
(mitigate)g(against)g(its)-182 2481 y(usefulness)d(in)i(modeling.)-182
2712 y Fl(6)99 b(Logistic)25 b(r)n(egr)n(ession)-182
2927 y Fd(6.1)90 b(Intr)n(oduction)-83 3133 y Fj(Logistic)25
b(re)o(gression)f(is)i(a)f(suitable)g(technique)f(for)g(the)i(prob-)
-182 3233 y(lem)19 b(of)h(classifying)f(the)g(modules)g(into)g(tw)o(o)h
(classes.)26 b(The)19 b(output)-182 3332 y(of)i(the)g(resulting)g
(equation)f(e)o(xpresses)h(the)g(probability)e(of)j(mem-)-182
3432 y(bership)15 b(to)h(the)h(second)e(group)g(\(with)h(one)f(minus)h
(this)h(probability)-182 3532 y(being)i(that)h(of)g(the)g(observ)n
(ation)e(belonging)g(to)i(the)h(\002rst)g(group\).)-83
3631 y(The)j(predictions)e(of)i(probabilities)e(can)i(be)g(turned)e
(into)i(clas-)-182 3731 y(si\002cation)i(by)f(using)h(a)h(threshold,)f
(usually)f(0.5)h(\(corresponding)-182 3831 y(to)21 b(50\045,)f(i.e.)28
b(the)21 b(most)g(lik)o(ely)f(class\))i(although)d(the)i(actual)g(v)n
(alue)-182 3930 y(used)g(depends)g(on)h(the)g(cost)g(of)g(incorrect)e
(decisions.)30 b(F)o(or)22 b(e)o(xam-)-182 4030 y(ple,)h(if)h
(incorrect)d(classi\002cations)j(to)f(the)g(\002rst)h(group)e(were)h
(more)-182 4129 y(e)o(xpensi)n(v)o(e)16 b(than)i(to)h(the)f(second)g
(group)f(the)h(cut-of)n(f)f(point)h(may)g(be)-182 4229
y(lo)n(wered)j(to)i(mo)o(v)o(e)e(some)h(of)h(the)f(more)g(mar)o(ginal)f
(decisions)h(into)-182 4329 y(the)h(second)f(class.)35
b(In)23 b(the)g(absence)f(of)h(an)o(y)f(guidelines)g(here)h(the)-182
4428 y(def)o(ault)c(v)n(alue)h(of)f(0.5)h(is)h(used.)-83
4528 y(When)46 b(de)n(v)o(eloping)d(the)j(logistic)f(re)o(gression)g
(models)g(the)-182 4628 y(method)18 b(used)h(w)o(as)h(to)f(enter)g(all)
h(\002)n(v)o(e)f(predictor)f(v)n(ariables)g(\()p Fk(Links)p
Fj(,)-182 4727 y Fk(Size)p Fj(,)27 b Fk(T)-6 b(ype)p
Fj(,)26 b Fk(Modi\002es)p Fj(,)h(and)e Fk(Entry)p Fj(\))g(and)g(then)g
(pick)g(out)h(the)f(one)-182 4827 y(or)30 b(tw)o(o)h(most)f
(signi\002cant)g(for)g(further)f(in)m(v)o(estigation.)53
b(Assum-)-182 4926 y(ing)23 b(that)h(tw)o(o)g(v)n(ariables)f(were)g
(reasonably)f(related)h(to)h Fk(Estimate)p Fj(,)-182
5026 y(these)18 b(v)n(ariables)f(were)g(then)h(used)f(indi)n(vidually)
-5 b(,)16 b(in)h(combination,)-182 5126 y(and)i(then)h(in)g
(combination)e(with)j(an)f(interaction.)-83 5225 y(Finally)15
b(the)h(model)e(with)h(the)h(one)e(or)h(tw)o(o)h(v)n(ariables)e(\(b)n
(ut)h(with-)-182 5325 y(out)21 b(the)i(interaction\))d(w)o(as)j(tested)
f(against)g(the)g(full)g(model)f(to)h(see)1974 83 y(if)g(an)o(y)e(of)h
(the)h(other)e(v)n(ariables)h(could)g(be)g(in\003uential.)28
b(If)21 b(this)h(test)1974 183 y(w)o(as)28 b(not)e(signi\002cant,)i
(best)f(indicated)f(by)g(the)h(f)o(act)g(that)g(the)g(dif-)1974
282 y(ference)i(in)i(de)n(viance)f(w)o(ould)g(be)g(insigni\002cant)g(e)
n(v)o(en)g(with)h(one)1974 382 y(de)o(gree)e(of)i(freedom,)g(then)f
(the)h(analysis)g(ended)f(there)g(and)g(the)1974 482
y(best)20 b(model)e(w)o(as)i(selected.)25 b(If)19 b(the)h(ef)n(fect)f
(of)g(the)g(remaining)f(v)n(ari-)1974 581 y(ables)24
b(w)o(as)h(signi\002cant)f(or)f(almost)h(signi\002cant)g(then)g(each)f
(of)h(the)1974 681 y(remaining)e(v)n(ariables)h(were)g(tested)h(to)g
(see)h(if)f(an)o(y)e(one)i(could)e(re-)1974 780 y(duce)j(the)i(de)n
(viance)d(signi\002cantly)-5 b(.)42 b(The)26 b(process)f(w)o(as)i
(repeated)1974 880 y(as)21 b(necessary)e(to)i(see)f(which)g(v)n
(ariables)f(were)h(useful.)2073 995 y(Thus)g(the)g(process)g(may)g(be)g
(summarized)f(as:)2078 1206 y(1.)40 b(run)18 b(the)h(analysis)f(using)g
(all)h(\002)n(v)o(e)g(predictor)d(v)n(ariables)i(with)2181
1306 y Fk(Estimate)j Fj(as)g(the)f(dependent)e(v)n(ariable)2078
1532 y(2.)40 b(select)17 b(all)g(v)n(ariables)e(that)h(ha)n(v)o(e)f
(signi\002cance)g(le)n(v)o(els)h(of)g(less)2181 1631
y(than)k(5\045)2078 1858 y(3.)40 b(add)20 b(v)n(ariables)f(that)i(are)f
(close)g(to)h(being)e(signi\002cant)2078 2084 y(4.)40
b(de)n(v)o(elop)19 b(models)g(using)2230 2310 y(\(a\))41
b(each)18 b(of)g(the)h(signi\002cant)f(or)g(nearly)f(signi\002cant)h(v)
n(ari-)2364 2410 y(ables)i(separately)2226 2573 y(\(b\))40
b(all)21 b(of)f(these)g(v)n(ariables)f(included)2230
2736 y(\(c\))41 b(all)21 b(of)f(these)g(v)n(ariables)f(plus)i(their)f
(interaction\(s\))2078 2962 y(5.)40 b(test)17 b(the)g(signi\002cance)e
(of)h(adding)e(the)j(remaining)d(v)n(ariables)2078 3188
y(6.)40 b(if)20 b(these)f(v)n(ariables)g(pro)o(vide)e(a)j
(signi\002cant)e(or)h(close)h(to)f(sig-)2181 3288 y(ni\002cant)h
(result)g(then)2230 3514 y(\(a\))41 b(test)21 b(adding)e(each)h
(separately)2226 3677 y(\(b\))40 b(add)e(the)g(signi\002cant)g(v)n
(ariable\(s\))f(and)h(repeat)g(the)2364 3777 y(process)2078
4003 y(7.)i(repeat)20 b(until)g(no)g(further)e(v)n(ariables)i(can)g(be)
g(added.)2073 4214 y(As)g(well)g(as)g(considering)d(the)i
(signi\002cance)f(of)h(changes)f(in)h(the)1974 4314 y(de)n(viance)33
b(statistics,)40 b(Akaik)o(e')-5 b(s)34 b(Information)e(Criterion)i
(\(AIC\))1974 4413 y(w)o(as)e(also)g(used)g(to)g(encourage)d
(parsimonious)h(models.)59 b(Ho)n(w-)1974 4513 y(e)n(v)o(er)m(,)28
b(where)f(small)h(dif)n(ferences)d(in)j(AIC)g(e)o(xisted)f(the)h(de)n
(viance)1974 4613 y(statistic)i(w)o(as)f(used)g(to)g(select)g(between)f
(the)h(simpler)f(and)g(more)1974 4712 y(comple)o(x)18
b(model.)2073 4827 y(Since)28 b(the)f(problem)e(as)j(it)f(stands)g
(deals)h(with)f(three)f(classes,)1974 4926 y(and)f(ordinary)f(logistic)
i(re)o(gression)f(is)i(only)e(suitable)h(for)f(binary)1974
5026 y(classi\002cation)g(a)h(modi\002cation)e(of)h(the)g(goal)g(is)h
(required)e(\(nomi-)1974 5126 y(nal)j(and)f(ordinal)g(logistic)h(re)o
(gression)e(are)i(other)f(options)g(here\).)1974 5225
y(Logistic)e(re)o(gression)e(models)h(were)h(used)g(to)g(separately)f
(predict)1974 5325 y(three)d(v)n(ariables,)f(de\002ned)g(as)i(sho)n(wn)
e(in)i(T)-7 b(able)20 b(9.)1859 5574 y(6)p eop
%%Page: 7 7
7 6 bop 29 3 3703 4 v 27 103 4 100 v 79 73 a Fj(Model)p
1456 103 V 1210 w(De)n(viance)p 1867 103 V 112 w(df)p
2050 103 V 99 w(P)o(arameters)p 2517 103 V 225 w(AIC)p
2886 103 V 239 w Fg(\037)3129 43 y Fc(2)p 3214 103 V
3266 73 a Fj(df)p 3383 103 V 99 w(p-v)n(alue)p 3730 103
V 29 106 3703 4 v 27 206 4 100 v 79 176 a(Constant)p
1456 206 V 1174 w(104.539)p 1867 206 V 96 w(76)p 2050
206 V 425 w(1)p 2517 206 V 99 w(106.539)p 2886 206 V
3214 206 V 3383 206 V 3730 206 V 27 306 V 79 276 a(Size)p
1456 306 V 1369 w(73.384)p 1867 306 V 96 w(74)p 2050
306 V 425 w(3)p 2517 306 V 141 w(79.384)p 2886 306 V
97 w(31.155)p 3214 306 V 124 w(2)p 3383 306 V 118 w(0.0000)p
3730 306 V 27 405 V 79 375 a(T)-7 b(ype)p 1456 405 V
1347 w(99.734)p 1867 405 V 96 w(75)p 2050 405 V 425 w(2)p
2517 405 V 99 w(103.734)p 2886 405 V 138 w(4.806)p 3214
405 V 125 w(1)p 3383 405 V 118 w(0.0284)p 3730 405 V
27 505 V 79 475 a(Size)20 b(+)h(T)-7 b(ype)p 1456 505
V 1116 w(69.641)p 1867 505 V 96 w(73)p 2050 505 V 425
w(4)p 2517 505 V 141 w(77.641)p 2886 505 V 97 w(34.898)p
3214 505 V 124 w(3)p 3383 505 V 118 w(0.0000)p 3730 505
V 27 604 V 79 575 a(Size)20 b(+)h(T)-7 b(ype)20 b(+)g
(Interaction\(Size,)e(T)-7 b(ype\))p 1456 604 V 264 w(66.622)p
1867 604 V 96 w(71)p 2050 604 V 425 w(6)p 2517 604 V
141 w(78.622)p 2886 604 V 97 w(37.917)p 3214 604 V 124
w(5)p 3383 604 V 118 w(0.0000)p 3730 604 V 27 704 V 79
674 a(Size)20 b(+)h(T)-7 b(ype)20 b(+)g(Entry)g(+)g(Links)g(+)h
(Modi\002es)p 1456 704 V 182 w(67.399)p 1867 704 V 96
w(70)p 2050 704 V 425 w(7)p 2517 704 V 141 w(81.399)p
2886 704 V 97 w(37.140)p 3214 704 V 124 w(6)p 3383 704
V 118 w(0.0000)p 3730 704 V 29 707 3703 4 v 477 859 a
Fe(T)-7 b(ab)o(le)24 b(10.)f(The)g(logistic)g(regression)g(models)g(f)n
(or)g(identifying)f(underestimates)p 590 1013 2580 4
v 588 1113 4 100 v 640 1083 a Fj(T)-6 b(erm)p 1968 1113
V 1204 w(De)n(viance)19 b(Change)p 2653 1113 V 98 w(df)p
2822 1113 V 99 w(p-v)n(alue)p 3168 1113 V 590 1116 2580
4 v 588 1216 4 100 v 640 1186 a(Size)p 1968 1216 V 1594
w(31.155)p 2653 1216 V 124 w(2)p 2822 1216 V 118 w(0.0000)p
3168 1216 V 588 1315 V 640 1286 a(T)-7 b(ype)p 1968 1315
V 1613 w(4.805)p 2653 1315 V 125 w(1)p 2822 1315 V 118
w(0.0284)p 3168 1315 V 588 1415 V 640 1385 a(Size)21
b Ff(j)g Fj(T)-7 b(ype)p 1968 1415 V 1364 w(30.093)p
2653 1415 V 124 w(2)p 2822 1415 V 118 w(0.0000)p 3168
1415 V 588 1515 V 640 1485 a(T)g(ype)20 b Ff(j)h Fj(Size)p
1968 1515 V 1406 w(3.743)p 2653 1515 V 125 w(1)p 2822
1515 V 118 w(0.0530)p 3168 1515 V 588 1614 V 640 1584
a(Interaction\(Size,)d(T)-7 b(ype\))p Ff(j)p Fj(Size)19
b(and)h(T)-7 b(ype)p 1968 1614 V 522 w(3.019)p 2653 1614
V 125 w(2)p 2822 1614 V 118 w(0.2210)p 3168 1614 V 588
1714 V 640 1684 a(Entry)i(,)19 b(Links,)g(Modi\002es)h
Ff(j)h Fj(Size)g(and)e(T)-7 b(ype)p 1968 1714 V 498 w(2.242)p
2653 1714 V 125 w(3)p 2822 1714 V 118 w(0.5237)p 3168
1714 V 590 1717 2580 4 v -76 1869 a Fe(T)g(ab)o(le)23
b(11.)h(The)e(anal)o(ysis)i(of)e(de)o(viance)h(f)n(or)g(the)g(logistic)
g(regression)g(models)g(f)n(or)g(identifying)f(underestimates)p
798 2023 2164 4 v 796 2123 4 100 v 1276 2123 V 1327 2093
a Fj(Predicted)d(0)p 1751 2123 V 100 w(Predicted)g(1)p
2227 2123 V 99 w(Percentage)g(Correct)p 2960 2123 V 798
2126 2164 4 v 796 2226 4 100 v 848 2196 a(Observ)o(ed)g(0)p
1276 2226 V 392 w(34)p 1751 2226 V 391 w(11)p 2227 2226
V 546 w(75.56)p 2960 2226 V 796 2325 V 848 2296 a(Observ)o(ed)g(1)p
1276 2325 V 433 w(6)p 1751 2325 V 392 w(26)p 2227 2325
V 546 w(81.25)p 2960 2325 V 798 2329 2164 4 v 647 2481
a Fe(T)-7 b(ab)o(le)24 b(12.)f(The)g(c)n(lassi\002cation)g(tab)o(le)g
(f)n(or)g(the)g(underestimates)g(model)-182 2814 y Fd(6.2)90
b(Model)22 b(1)h(\226)f(under)n(estimates)-83 3011 y
Fj(The)k(\002rst)i(model)d(for)h(predicting)f(underestimates)g
(produced)-182 3111 y(the)c(results)h(sho)n(wn)f(in)g(T)-7
b(able)22 b(10.)28 b(From)21 b(this)h(table)g(it)g(is)g(possible)-182
3211 y(to)j(construct)f(T)-7 b(able)25 b(11)g(which)g(sho)n(ws)g(the)g
(change)f(in)h(de)n(viance)-182 3310 y(for)19 b(each)h(term.)-83
3410 y(From)f(T)-7 b(ables)19 b(10)f(and)h(11)f(it)i(can)f(be)g(seen)g
(that)g(the)g(best)g(model)-182 3509 y(is)30 b Fk(Size)f
Fj(and)f Fk(T)-6 b(ype)p Fj(.)52 b(Adding)27 b Fk(Size)j
Fj(to)f Fk(T)-6 b(ype)29 b Fj(is)h(certainly)e(signif-)-182
3609 y(icant)e(and)f(adding)g(the)i(v)n(ariables)e(in)h(the)h(re)n(v)o
(erse)e(order)g(still)i(re-)-182 3709 y(sults)17 b(in)g(a)g(p-v)n(alue)
f(of)g(0.0530)f(for)h Fk(T)-6 b(ype)p Fj(.)23 b(This)17
b(is)h(close)f(enough)e(to)-182 3808 y(use,)25 b(especially)e(since)i
(AIC)f(is)h(also)g(minimized)e(for)g(this)i(model)-182
3908 y(\(77.641\).)52 b(Adding)29 b(an)i(interaction)e(to)h(this)h
(model,)h(or)e(the)g(re-)-182 4008 y(maining)25 b(three)h(v)n(ariables)
g(does)h(not)f(impro)o(v)o(e)f(the)h(model)g(suf)n(\002-)-182
4107 y(ciently)-5 b(.)23 b(The)c(performance)d(of)i(the)h(model)f(is)i
(sho)n(wn)e(in)h(T)-7 b(able)19 b(12)-182 4207 y(with)h(a)h(cut-of)n(f)
d(point)i(of)g(0.5.)-148 4442 y Fi(lo)l(git)8 b Fh(\()p
Fi(Under)h Fh(\))24 b(=)f Ff(\000)p Fh(0)p Fg(:)p Fh(4989)15
b(+)j(1)p Fg(:)p Fh(3093)p Fi(T)-6 b(yp)l(e)23 b Ff(\000)18
b Fh(4)p Fg(:)p Fh(2847)p Fi(Size1)732 4567 y Ff(\000)g
Fh(1)p Fg(:)p Fh(4696)p Fi(Size2)1690 4675 y Fj(\(1\))-83
4827 y(The)h(model)g(is)h(therefore)d(as)j(sho)n(wn)f(in)g(Equation)f
(1.)25 b(This)19 b(can)-182 4926 y(be)30 b(interpreted)f(as)j(meaning)d
(that)i(for)g(a)g(lar)o(ge)f(\()p Fk(Size3)p Fj(\))g(screen-)-182
5026 y(based)c(module)f(the)h(probability)f(of)h(the)h(estimate)g
(being)e(an)i(un-)-182 5126 y(derestimate)34 b(is)j(about)d(69\045)i
(\(note)e(that)i(screens)g(are)f Fk(T)-6 b(ype)p Fj(=1,)-182
5225 y(and)31 b(reports)f(are)i Fk(T)-6 b(ype)p Fj(=2\).)58
b(F)o(or)31 b(a)h(report)f(the)h(probability)d(in-)-182
5325 y(creases)20 b(\(to)f(89\045\),)g(while)h(for)g(smaller)f(modules)
g(the)h(probability)1974 2814 y(decreases)i(\(to)h(3\045)g(and)f
(34\045)h(respecti)n(v)o(ely)e(for)h Fk(Size1)h Fj(and)f
Fk(Size2)1974 2914 y Fj(screens,)e(and)f(10\045)h(and)g(66\045)g(for)f
Fk(Size1)h Fj(and)g Fk(Size2)g Fj(reports\).)2073 3014
y(Thus)29 b(it)g(can)g(be)g(seen)f(that)h(the)g(size)h(and)e(type)g(of)
h(the)f(mod-)1974 3113 y(ule)k(do)g(appear)f(to)h(share)g(an)g
(association)g(with)g(the)g(quality)f(of)1974 3213 y(the)18
b(estimate)h(in)g(terms)g(of)f(project)g(management)e(making)i(under)n
(-)1974 3313 y(estimates.)43 b(More)26 b(speci\002cally)-5
b(,)26 b(reports)g(are)g(more)f(lik)o(ely)h(to)g(be)1974
3412 y(underestimated)18 b(as)j(are)f(lar)o(ger)f(modules.)1974
3621 y Fd(6.3)90 b(Model)23 b(2)f(\226)h(err)n(ors)2073
3830 y Fj(The)k(second)f(logistic)i(re)o(gression)d(model)h(for)h
(predicting)e(er)n(-)1974 3930 y(rors)k(in)h(estimates)h(produced)c
(the)j(results)g(sho)n(wn)f(in)h(T)-7 b(able)30 b(13.)1974
4029 y(Using)e(these)g(results)g(T)-7 b(able)28 b(14)g(can)g(be)g
(obtained)e(sho)n(wing)h(the)1974 4129 y(change)j(in)h(de)n(viance)f
(for)h(each)g(term.)58 b(The)31 b(lack)g(of)g(data)g(pre-)1974
4229 y(v)o(ented)15 b(the)i(de)n(v)o(elopment)d(of)i(a)h(model)f
(including)f(the)i(interaction)1974 4328 y(between)27
b Fk(Links)i Fj(and)f Fk(Size)g Fj(e)n(v)o(en)g(if)g
Fk(Size)h Fj(w)o(as)g(considered)d(suf)n(\002-)1974 4428
y(ciently)k(signi\002cant)h(to)g(w)o(arrant)f(further)f(in)m(v)o
(estigation.)55 b(There)1974 4528 y(are)20 b(zero)g(links)g(to)g(other)
g(tables)g(only)f(for)h(small)h(modules.)2073 4628 y(From)e(T)-7
b(ables)19 b(13)g(and)f(14)h(it)g(can)g(be)g(seen)g(that)g(the)g(best)g
(model)1974 4727 y(is)k Fk(Links)p Fj(.)33 b(This)22
b(does)h(not)f(quite)g(minimize)g(AIC)h(\(88.634)d(v)o(ersus)1974
4827 y(88.475)k(for)i(both)g Fk(Size)h Fj(and)f Fk(Links)p
Fj(\))g(b)n(ut)h(is)g(v)o(ery)e(close.)45 b(Adding)1974
4926 y(other)25 b(v)n(ariables)h(to)g Fk(Links)h Fj(does)f(not)g(impro)
o(v)o(e)e(the)i(model)f(suf)n(\002-)1974 5026 y(ciently)-5
b(.)24 b(The)18 b(performance)e(of)j(the)g(model)f(is)h(sho)n(wn)g(in)g
(T)-7 b(able)19 b(15)1974 5126 y(with)k(a)g(cut-of)n(f)f(point)g(of)h
(0.5.)32 b(Since)23 b(the)g(model)g(only)f(produced)1974
5225 y(tw)o(o)27 b(v)n(alues)g(\(0.6897)e(and)i(0.8947\))e(taking)h(an)
o(y)g(v)n(alue)h(between)1974 5325 y(these)20 b(tw)o(o)h(results)f(in)h
(the)f(performance)d(sho)n(wn)i(in)i(T)-7 b(able)20 b(16)1859
5574 y(7)p eop
%%Page: 8 8
8 7 bop 70 3 3620 4 v 68 103 4 100 v 120 73 a Fj(Model)p
1497 103 V 1211 w(De)n(viance)p 1909 103 V 112 w(df)p
2091 103 V 99 w(P)o(arameters)p 2558 103 V 183 w(AIC)p
2886 103 V 197 w Fg(\037)3087 43 y Fc(2)p 3173 103 V
3224 73 a Fj(df)p 3342 103 V 99 w(p-v)n(alue)p 3688 103
V 70 106 3620 4 v 68 206 4 100 v 120 176 a(Constant)p
1497 206 V 1216 w(88.209)p 1909 206 V 97 w(76)p 2091
206 V 425 w(1)p 2558 206 V 99 w(90.209)p 2886 206 V 3173
206 V 3342 206 V 3688 206 V 68 306 V 120 276 a(Size)p
1497 306 V 1369 w(86.218)p 1909 306 V 97 w(74)p 2091
306 V 425 w(3)p 2558 306 V 99 w(92.218)p 2886 306 V 97
w(1.991)p 3173 306 V 125 w(2)p 3342 306 V 118 w(0.3696)p
3688 306 V 68 405 V 120 375 a(Links)p 1497 405 V 1322
w(84.634)p 1909 405 V 97 w(75)p 2091 405 V 425 w(2)p
2558 405 V 99 w(88.634)p 2886 405 V 97 w(3.574)p 3173
405 V 125 w(1)p 3342 405 V 118 w(0.0587)p 3688 405 V
68 505 V 120 475 a(Size)21 b(+)g(Links)p 1497 505 V 1090
w(80.475)p 1909 505 V 97 w(73)p 2091 505 V 425 w(4)p
2558 505 V 99 w(88.475)p 2886 505 V 97 w(7.734)p 3173
505 V 125 w(3)p 3342 505 V 118 w(0.0518)p 3688 505 V
68 604 V 120 575 a(Size)g(+)g(T)-7 b(ype)19 b(+)i(Entry)e(+)i(Links)f
(+)g(Modi\002es)p 1497 604 V 182 w(80.118)p 1909 604
V 97 w(70)p 2091 604 V 425 w(7)p 2558 604 V 99 w(94.118)p
2886 604 V 97 w(8.091)p 3173 604 V 125 w(6)p 3342 604
V 118 w(0.2315)p 3688 604 V 70 608 3620 4 v 414 760 a
Fe(T)-7 b(ab)o(le)24 b(13.)f(The)g(logistic)g(regression)g(models)g(f)n
(or)g(identifying)f(err)n(or)o(s)i(in)f(estimates)p 590
914 2580 4 v 588 1013 4 100 v 640 983 a Fj(T)-6 b(erm)p
1968 1013 V 1204 w(De)n(viance)19 b(Change)p 2653 1013
V 98 w(df)p 2822 1013 V 99 w(p-v)n(alue)p 3168 1013 V
590 1017 2580 4 v 588 1116 4 100 v 640 1086 a(Size)p
1968 1116 V 1635 w(1.991)p 2653 1116 V 125 w(2)p 2822
1116 V 118 w(0.3695)p 3168 1116 V 588 1216 V 640 1186
a(Links)p 1968 1216 V 1588 w(3.575)p 2653 1216 V 125
w(1)p 2822 1216 V 118 w(0.0284)p 3168 1216 V 588 1315
V 640 1286 a(Size)i Ff(j)g Fj(Links)p 1968 1315 V 1380
w(4.159)p 2653 1315 V 125 w(2)p 2822 1315 V 118 w(0.1250)p
3168 1315 V 588 1415 V 640 1385 a(Links)f Ff(j)h Fj(Size)p
1968 1415 V 1381 w(5.743)p 2653 1415 V 125 w(1)p 2822
1415 V 118 w(0.0166)p 3168 1415 V 588 1515 V 640 1485
a(Entry)-5 b(,)19 b(T)-7 b(ype,)19 b(Modi\002es)h Ff(j)h
Fj(Size)g(and)e(Links)p 1968 1515 V 498 w(0.357)p 2653
1515 V 125 w(3)p 2822 1515 V 159 w(0.949)p 3168 1515
V 590 1518 2580 4 v -83 1670 a Fe(T)-7 b(ab)o(le)16 b(14.)g(The)f(anal)
o(ysis)h(of)f(de)o(viance)h(f)n(or)f(the)g(logistic)g(regression)h
(models)f(f)n(or)h(identifying)e(err)n(or)o(s)j(in)e(estimates)p
798 1824 2164 4 v 796 1924 4 100 v 1276 1924 V 1327 1894
a Fj(Predicted)k(0)p 1751 1924 V 100 w(Predicted)g(1)p
2227 1924 V 99 w(Percentage)g(Correct)p 2960 1924 V 798
1927 2164 4 v 796 2027 4 100 v 848 1997 a(Observ)o(ed)g(0)p
1276 2027 V 433 w(0)p 1751 2027 V 392 w(20)p 2227 2027
V 588 w(0.00)p 2960 2027 V 796 2126 V 848 2096 a(Observ)o(ed)g(1)p
1276 2126 V 433 w(0)p 1751 2126 V 392 w(57)p 2227 2126
V 505 w(100.00)p 2960 2126 V 798 2130 2164 4 v 607 2281
a Fe(T)-7 b(ab)o(le)23 b(15.)h(The)f(c)n(lassi\002cation)g(tab)o(le)g
(f)n(or)g(the)g(err)n(or)o(s)h(in)f(estimate)g(model)p
798 2419 V 796 2519 4 100 v 1276 2519 V 1327 2489 a Fj(Predicted)c(0)p
1751 2519 V 100 w(Predicted)g(1)p 2227 2519 V 99 w(Percentage)g
(Correct)p 2960 2519 V 798 2522 2164 4 v 796 2622 4 100
v 848 2592 a(Observ)o(ed)g(0)p 1276 2622 V 392 w(18)p
1751 2622 V 433 w(2)p 2227 2622 V 546 w(90.00)p 2960
2622 V 796 2721 V 848 2691 a(Observ)o(ed)g(1)p 1276 2721
V 392 w(40)p 1751 2721 V 391 w(17)p 2227 2721 V 546 w(29.82)p
2960 2721 V 798 2725 2164 4 v 349 2877 a Fe(T)-7 b(ab)o(le)23
b(16.)h(The)e(c)n(lassi\002cation)i(tab)o(le)f(f)n(or)g(the)f(err)n(or)
o(s)j(in)d(estimate)h(model)g(\(cutoff)e(0.74\))148 3381
y Fi(lo)l(git)8 b Fh(\()p Fi(Err)l(or)h Fh(\))25 b(=)d(2)p
Fg(:)p Fh(1401)16 b Ff(\000)i Fh(1)p Fg(:)p Fh(3416)p
Fi(Links)239 b Fj(\(2\))-83 3537 y(The)29 b(model)f(is)h(therefore)e
(as)j(sho)n(wn)e(in)h(Equation)e(2.)51 b(F)o(or)28 b(a)-182
3636 y(module)e(with)i(no)g(links)g(to)g(other)f(tables)h(the)g
(probability)e(of)i(an)-182 3736 y(error)g(in)i(estimation)g(of)f
(greater)g(than)g(35\045)h(is)h(89\045,)g(and)f(with)-182
3836 y(links)c(this)g(drops)f(to)h(69\045.)42 b(Thus)25
b(it)i(can)e(be)h(seen)g(that)g(the)g(inci-)-182 3935
y(dence)f(of)i(links)f(to)h(other)f(tables)h(from)f(the)g(main)h(table)
f(is)i(asso-)-182 4035 y(ciated)c(with)g(more)g(accurate)f(estimation.)
37 b(Thus)24 b(more)g(comple)o(x)-182 4135 y(and)h(lar)o(ger)g(modules)
g(seem)h(to)g(ha)n(v)o(e)g(been)f(estimated)h(more)f(ac-)-182
4234 y(curately)-5 b(,)25 b(which)h(could)f(suggest)g(that)h(more)f
(care)h(w)o(as)h(tak)o(en)e(of)-182 4334 y(their)d(estimates)g(or)g
(that)g(more)g(care)g(w)o(as)h(tak)o(en)e(with)i(their)f(man-)-182
4433 y(agement)d(to)h(k)o(eep)g(them)f(to)i(schedule.)-182
4630 y Fd(6.4)90 b(Model)22 b(3)h(\226)f(o)o(v)o(er)n(estimates)-83
4827 y Fj(The)k(third)f(logistic)i(re)o(gression)d(model)i(for)f
(predicting)f(o)o(v)o(er)n(-)-182 4926 y(estimates)g(in)g(ef)n(fort)f
(produced)e(the)j(results)g(sho)n(wn)g(in)g(T)-7 b(able)24
b(17.)-182 5026 y(From)d(here)f(T)-7 b(able)22 b(18)f(can)g(be)h
(obtained)e(sho)n(wing)g(the)i(change)e(in)-182 5126
y(de)n(viance)e(for)i(each)g(term.)-83 5225 y(From)f(T)-7
b(ables)19 b(17)f(and)h(18)f(it)i(can)f(be)g(seen)g(that)g(the)g(best)g
(model)-182 5325 y(is)26 b Fk(Size)p Fj(,)i Fk(T)-6 b(ype)p
Fj(,)27 b(and)e Fk(Entry)p Fj(.)43 b(The)25 b(addition)g(of)h
Fk(Links)g Fj(and)f Fk(Modi-)1974 3226 y(\002es)f Fj(does)g(not)g(seem)
g(to)g(impro)o(v)o(e)e(estimation.)36 b(This)25 b(model)e(also)1974
3326 y(minimizes)f(AIC)h(\(63.687\).)29 b(The)22 b(performance)d(of)j
(the)h(model)e(is)1974 3425 y(sho)n(wn)e(in)i(T)-7 b(able)20
b(19)g(with)g(a)h(cut-of)n(f)d(point)i(of)f(0.5.)2022
3828 y Fi(lo)l(git)8 b Fh(\()p Fi(Over)h Fh(\))24 b(=)e
Ff(\000)p Fh(1)p Fg(:)p Fh(7646)16 b(+)i(3)p Fg(:)p Fh(9111)p
Fi(Size1)28 b Fh(+)18 b(1)p Fg(:)p Fh(1522)p Fi(Size2)2853
3953 y Ff(\000)g Fh(2)p Fg(:)p Fh(9399)p Fi(T)-6 b(yp)l(e)23
b Fh(+)18 b(2)p Fg(:)p Fh(6532)p Fi(Entry)3846 4061 y
Fj(\(3\))2073 4381 y(The)32 b(actual)f(model)f(is)j(the)e(one)g(sho)n
(wn)g(in)g(Equation)f(3.)58 b(In)1974 4481 y(this)19
b(case)f(the)g(smaller)h(modules)e(are)h(more)f(lik)o(ely)h(to)g(be)h
(o)o(v)o(eresti-)1974 4580 y(mated,)f(as)i(are)e(modules)g(for)g(data)h
(entry)-5 b(.)23 b(Reports)c(are)f(less)i(lik)o(ely)1974
4680 y(to)g(be)g(o)o(v)o(erestimated)e(than)i(screens.)2073
4827 y(This)d(model)e(is)j(consistent)e(with)g(that)g(obtained)f(for)h
(predicting)1974 4926 y(underestimates)21 b(where)h(this)i(w)o(as)f
(associated)g(with)g(lar)o(ger)e(mod-)1974 5026 y(ules)i(that)h(were)f
(screen)g(based.)34 b(Here)23 b(ho)n(we)n(v)o(er)f(a)h(ne)n(w)h(v)n
(ariable,)1974 5126 y Fk(Entry)p Fj(,)29 b(has)e(emer)o(ged)e(as)j
(useful)f(for)f(classi\002cation.)47 b(This)27 b(v)n(ari-)1974
5225 y(able)33 b(w)o(as)h(only)e(the)h(third)f(in)i(the)f(entry)f
(procedure,)h(and)g(only)1974 5325 y(seems)21 b(to)f(be)g
(in\003uential)f(after)h(accounting)e(for)i Fk(Size)g
Fj(and)g Fk(T)-6 b(ype)p Fj(.)1859 5574 y(8)p eop
%%Page: 9 9
9 8 bop 50 3 3661 4 v 48 103 4 100 v 99 73 a Fj(Model)p
1477 103 V 1211 w(De)n(viance)p 1888 103 V 112 w(df)p
2070 103 V 99 w(P)o(arameters)p 2538 103 V 183 w(AIC)p
2866 103 V 239 w Fg(\037)3108 43 y Fc(2)p 3194 103 V
3245 73 a Fj(df)p 3362 103 V 99 w(p-v)n(alue)p 3709 103
V 50 106 3661 4 v 48 206 4 100 v 99 176 a(Constant)p
1477 206 V 1216 w(97.073)p 1888 206 V 97 w(76)p 2070
206 V 425 w(1)p 2538 206 V 99 w(99.073)p 2866 206 V 3194
206 V 3362 206 V 3709 206 V 48 306 V 99 276 a(Size)p
1477 306 V 1369 w(66.375)p 1888 306 V 97 w(74)p 2070
306 V 425 w(3)p 2538 306 V 99 w(72.375)p 2866 306 V 97
w(30.698)p 3194 306 V 125 w(2)p 3362 306 V 117 w(0.0000)p
3709 306 V 48 405 V 99 375 a(T)-7 b(ype)p 1477 405 V
1347 w(86.508)p 1888 405 V 97 w(75)p 2070 405 V 425 w(2)p
2538 405 V 99 w(90.508)p 2866 405 V 97 w(10.565)p 3194
405 V 125 w(1)p 3362 405 V 117 w(0.0012)p 3709 405 V
48 505 V 99 475 a(Size)21 b(+)g(T)-7 b(ype)p 1477 505
V 1115 w(58.763)p 1888 505 V 97 w(73)p 2070 505 V 425
w(4)p 2538 505 V 99 w(66.763)p 2866 505 V 97 w(38.310)p
3194 505 V 125 w(3)p 3362 505 V 117 w(0.0000)p 3709 505
V 48 604 V 99 575 a(Size)21 b(+)g(T)-7 b(ype)20 b(+)g
(Interaction\(Size,)e(T)-7 b(ype\))p 1477 604 V 263 w(57.868)p
1888 604 V 97 w(71)p 2070 604 V 425 w(6)p 2538 604 V
99 w(69.868)p 2866 604 V 97 w(39.205)p 3194 604 V 125
w(5)p 3362 604 V 117 w(0.0000)p 3709 604 V 48 704 V 99
674 a(Size)21 b(+)g(T)-7 b(ype)20 b(+)g(Links)p 1477
704 V 838 w(54.112)p 1888 704 V 97 w(72)p 2070 704 V
425 w(5)p 2538 704 V 99 w(64.112)p 2866 704 V 97 w(42.961)p
3194 704 V 125 w(4)p 3362 704 V 117 w(0.0000)p 3709 704
V 48 804 V 99 774 a(Size)h(+)g(T)-7 b(ype)20 b(+)g(Modi\002es)p
1477 804 V 732 w(58.754)p 1888 804 V 97 w(72)p 2070 804
V 425 w(5)p 2538 804 V 99 w(68.754)p 2866 804 V 97 w(38.319)p
3194 804 V 125 w(4)p 3362 804 V 117 w(0.0000)p 3709 804
V 48 903 V 99 873 a(Size)h(+)g(T)-7 b(ype)20 b(+)g(Entry)p
1477 903 V 842 w(53.687)p 1888 903 V 97 w(72)p 2070 903
V 425 w(5)p 2538 903 V 99 w(63.687)p 2866 903 V 97 w(43.386)p
3194 903 V 125 w(4)p 3362 903 V 117 w(0.0000)p 3709 903
V 48 1003 V 99 973 a(Size)h(+)g(T)-7 b(ype)20 b(+)g(Entry)f(+)i(Links)f
(+)g(Modi\002es)p 1477 1003 V 182 w(52.352)p 1888 1003
V 97 w(70)p 2070 1003 V 425 w(7)p 2538 1003 V 99 w(66.352)p
2866 1003 V 97 w(44.721)p 3194 1003 V 125 w(6)p 3362
1003 V 117 w(0.0000)p 3709 1003 V 50 1006 3661 4 v 506
1158 a Fe(T)-7 b(ab)o(le)23 b(17.)h(The)f(logistic)f(regression)i
(models)f(f)n(or)g(identifying)f(o)n(verestimates)p 580
1312 2601 4 v 578 1412 4 100 v 630 1382 a Fj(T)-6 b(erm)p
1978 1412 V 1224 w(De)n(viance)20 b(Change)p 2663 1412
V 98 w(df)p 2832 1412 V 99 w(p-v)n(alue)p 3179 1412 V
580 1415 2601 4 v 578 1515 4 100 v 630 1485 a(Size)p
1978 1515 V 1614 w(30.698)p 2663 1515 V 124 w(2)p 2832
1515 V 118 w(0.0000)p 3179 1515 V 578 1614 V 630 1584
a(T)-7 b(ype)p 1978 1614 V 1592 w(10.565)p 2663 1614
V 124 w(1)p 2832 1614 V 118 w(0.0012)p 3179 1614 V 578
1714 V 630 1684 a(Size)20 b Ff(j)h Fj(T)-7 b(ype)p 1978
1714 V 1385 w(27.745)p 2663 1714 V 124 w(2)p 2832 1714
V 118 w(0.0000)p 3179 1714 V 578 1814 V 630 1784 a(T)g(ype)19
b Ff(j)i Fj(Size)p 1978 1814 V 1427 w(7.612)p 2663 1814
V 125 w(1)p 2832 1814 V 118 w(0.0058)p 3179 1814 V 578
1913 V 630 1883 a(Interaction\(Size,)c(T)-7 b(ype\))p
Ff(j)p Fj(Size)20 b(and)f(T)-7 b(ype)p 1978 1913 V 543
w(0.895)p 2663 1913 V 125 w(2)p 2832 1913 V 118 w(0.6392)p
3179 1913 V 578 2013 V 630 1983 a(Entry)19 b Ff(j)i Fj(Size)f(and)g(T)
-7 b(ype)p 1978 2013 V 1080 w(5.076)p 2663 2013 V 125
w(1)p 2832 2013 V 118 w(0.0243)p 3179 2013 V 578 2112
V 630 2083 a(Links)20 b Ff(j)g Fj(Size)h(and)f(T)-7 b(ype)p
1978 2112 V 1075 w(4.651)p 2663 2112 V 125 w(1)p 2832
2112 V 118 w(0.0310)p 3179 2112 V 578 2212 V 630 2182
a(Modi\002es)20 b Ff(j)g Fj(Size)h(and)f(T)-7 b(ype)p
1978 2212 V 969 w(0.000)p 2663 2212 V 125 w(1)p 2832
2212 V 118 w(1.0000)p 3179 2212 V 578 2312 V 630 2282
a(Links,)19 b(Modi\002es)h Ff(j)h Fj(Size,)g(Entry)-5
b(,)18 b(and)i(T)-7 b(ype)p 1978 2312 V 497 w(1.335)p
2663 2312 V 125 w(2)p 2832 2312 V 118 w(0.5130)p 3179
2312 V 580 2315 2601 4 v -48 2467 a Fe(T)g(ab)o(le)24
b(18.)f(The)g(anal)o(ysis)g(of)g(de)o(viance)g(f)n(or)g(the)g(logistic)
f(regression)i(models)f(f)n(or)g(identifying)e(o)n(verestimates)p
798 2621 2164 4 v 796 2721 4 100 v 1276 2721 V 1327 2691
a Fj(Predicted)e(0)p 1751 2721 V 100 w(Predicted)g(1)p
2227 2721 V 99 w(Percentage)g(Correct)p 2960 2721 V 798
2724 2164 4 v 796 2824 4 100 v 848 2794 a(Observ)o(ed)g(0)p
1276 2824 V 392 w(38)p 1751 2824 V 391 w(14)p 2227 2824
V 546 w(73.08)p 2960 2824 V 796 2923 V 848 2893 a(Observ)o(ed)g(1)p
1276 2923 V 433 w(3)p 1751 2923 V 392 w(22)p 2227 2923
V 546 w(88.00)p 2960 2923 V 798 2927 2164 4 v 676 3078
a Fe(T)-7 b(ab)o(le)23 b(19.)h(The)e(c)n(lassi\002cation)i(tab)o(le)f
(f)n(or)g(the)f(o)n(verestimates)i(model)-182 3412 y
Fd(6.5)90 b(Conclusions)21 b(fr)n(om)i(logistic)g(r)n(egr)n(ession)-83
3703 y Fj(From)32 b(the)g(abo)o(v)o(e)f(three)h(analyses)g(it)h
(appears)e(that)h(the)h(tw)o(o)-182 3803 y(main)16 b(f)o(actors)h(that)
g(are)g(associated)f(with)h(errors)f(in)i(estimation)e(are)-182
3902 y(the)21 b(size)h(and)f(type)f(of)h(the)h(module.)27
b(The)21 b(e)o(xistence)f(of)h(links)g(and)-182 4002
y(data)28 b(entry)h(capability)f(also)h(in\003uence)f(the)h(estimate)g
(accurac)o(y)-5 b(.)-182 4102 y(The)32 b(analysis)h(for)f(errors)g(is)i
(less)g(signi\002cant)f(than)f(for)g(under)n(-)-182 4201
y(and)e(o)o(v)o(erestimates)g(since)h(the)g(point)f(of)h(the)g
(analysis)g(is)h(to)f(be)-182 4301 y(able)18 b(to)h(mak)o(e)f(such)h
(directional)e(predictions.)23 b(Ho)n(we)n(v)o(er)m(,)17
b(it)i(is)h(in-)-182 4401 y(teresting)26 b(that)i(only)e
Fk(Links)h Fj(w)o(as)h(required)e(for)g(modeling)g(errors)-182
4500 y(in)d(prediction.)34 b(This,)25 b(along)e(with)h(the)f(emer)o
(gence)f(of)h(the)h Fk(Entry)-182 4600 y Fj(v)n(ariable)i(for)h(o)o(v)o
(erestimates,)g(suggests)g(the)h(models)f(for)f(under)n(-)-182
4699 y(and)19 b(o)o(v)o(erestimation)f(are)i(not)g(entirely)f
(symmetrical.)-83 4827 y(There)g(is)h(de\002nite)e(e)n(vidence)g(that)h
(the)h(type)e(of)h(the)g(module)f(af-)-182 4926 y(fects)29
b(the)f(probability)-5 b(,)29 b(which)f(is)h(an)g(interesting)f
(\002nding.)49 b(The)-182 5026 y(other)19 b(conclusion)g(that)i(size)g
(af)n(fects)f(estimate)h(accurac)o(y)e(is)j(more)-182
5126 y(predictable)29 b(since)i(small)h(modules)e(will)h(generally)f
(be)h(correct)-182 5225 y(or)d(o)o(v)o(erestimated,)h(while)f(lar)o(ge)
g(modules)g(will)h(more)f(often)g(be)-182 5325 y(underestimated.)1974
3412 y Fl(7)99 b(Log-linear)26 b(analysis)1974 3625 y
Fd(7.1)90 b(Intr)n(oduction)2073 3831 y Fj(After)23 b(the)g(contingenc)
o(y)e(table)i(analysis)g(and)f(logistic)h(re)o(gres-)1974
3930 y(sion)18 b(models,)f(three)g(v)n(ariables)g(were)h(selected)g
(for)f(further)f(analy-)1974 4030 y(sis.)26 b(These)20
b(are)g Fk(Size)p Fj(,)h Fk(T)-6 b(ype)p Fj(,)19 b(and)h
Fk(Estimate)p Fj(.)25 b(The)20 b(purpose)f(of)h(this)1974
4129 y(analysis)25 b(using)g(log-linear)e(modeling)g(is)j(to)g(in)m(v)o
(estigate)d(associ-)1974 4229 y(ations)28 b(between)g
Fk(Estimate)h Fj(and)f(le)n(v)o(els)g(of)h(the)f(other)g(v)n(ariables.)
1974 4329 y(The)d(adv)n(antage)e(of)i(log-linear)f(modeling)f(o)o(v)o
(er)h(logistic)h(re)o(gres-)1974 4428 y(sion)32 b(is)h(that)f(here)f
(an)o(y)g(number)f(of)i(output)f(cate)o(gories)f(can)i(be)1974
4528 y(used.)2073 4628 y(While)23 b Fk(Entry)f Fj(and)g
Fk(Links)g Fj(were)g(also)h(found)d(to)j(be)f(signi\002cant)1974
4727 y(in)h(the)h(logistic)f(re)o(gression)f(section,)i(the)o(y)e(ha)n
(v)o(e)h(been)f(discarded)1974 4827 y(here)30 b(for)g(three)h(reasons)f
(each.)56 b(First,)34 b(common)29 b(to)i(both)f(v)n(ari-)1974
4926 y(ables,)j(there)d(is)i(a)f(desire)g(for)f(simplicity)g(in)h(this)
g(analysis)g(that)1974 5026 y(is)24 b(greatly)e(aided)g(by)h
(restricting)f(the)h(analysis)g(to)g(just)h(these)f(tw)o(o)1974
5126 y(v)n(ariables.)k(P)o(arsimon)o(y)19 b(is)j(a)g(useful)e(goal)h
(to)g(k)o(eep)g(in)g(mind)g(when)1974 5225 y(de)n(v)o(eloping)c(models)
i(that)h(must)g(be)g(interpreted)f(and)g(relied)h(upon)1974
5325 y(by)g(managers)f(who)g(are)h(not)g(quali\002ed)g(statisticians.)
1859 5574 y(9)p eop
%%Page: 10 10
10 9 bop 724 3 2313 4 v 722 103 4 100 v 1113 103 V 1861
73 a Fj(Size)p 3035 103 V 722 203 V 1113 203 V 1359 173
a(1)510 b(2)g(3)p 3035 203 V 722 302 V 1113 302 V 1298
272 a(T)-7 b(ype)387 b(T)-7 b(ype)387 b(T)-7 b(ype)p
3035 302 V 722 402 V 1113 402 V 1237 372 a(1)223 b(2)h(1)244
b(2)h(1)224 b(2)160 b(T)-7 b(otal)p 3035 402 V 724 405
2313 4 v 722 505 4 100 v 774 475 a(Estimate)p 1113 505
V 1399 505 V 1644 505 V 1930 505 V 2217 505 V 2503 505
V 2748 505 V 3035 505 V 722 604 V 885 575 a(-1)p 1113
604 V 230 w(0.00)p 1399 604 V 118 w(1.00)p 1644 604 V
119 w(5.00)p 1930 604 V 140 w(7.00)p 2217 604 V 118 w(15.00)p
2503 604 V 98 w(4.00)p 2748 604 V 149 w(32)p 3035 604
V 722 704 V 898 674 a(0)p 1113 704 V 245 w(5.00)p 1399
704 V 118 w(1.00)p 1644 704 V 119 w(5.00)p 1930 704 V
140 w(4.00)p 2217 704 V 139 w(4.00)p 2503 704 V 119 w(1.00)p
2748 704 V 149 w(20)p 3035 704 V 722 804 V 898 774 a(1)p
1113 804 V 224 w(18.00)p 1399 804 V 97 w(1.00)p 1644
804 V 119 w(4.00)p 1930 804 V 140 w(0.00)p 2217 804 V
139 w(2.00)p 2503 804 V 119 w(0.00)p 2748 804 V 149 w(25)p
3035 804 V 724 807 2313 4 v 722 907 4 100 v 835 877 a(T)g(otal)p
1113 907 V 160 w(23.00)p 1399 907 V 97 w(3.00)p 1644
907 V 98 w(14.00)p 1930 907 V 98 w(11.00)p 2217 907 V
97 w(21.00)p 2503 907 V 98 w(5.00)p 2748 907 V 98 w(77.00)p
3035 907 V 724 910 2313 4 v 756 1062 a Fe(T)g(ab)o(le)23
b(20.)h(Overall)f(frequenc)o(y)g(of)g Fk(Size)p Fe(,)g
Fk(T)-6 b(ype)p Fe(,)22 b Fk(Estimate)h Fe(v)n(ariab)o(les)p
511 1216 2739 4 v 509 1315 4 100 v 899 1315 V 1772 1286
a Fj(Size)p 3248 1315 V 509 1415 V 899 1415 V 1194 1385
a(1)587 b(2)f(3)p 3248 1415 V 509 1515 V 899 1515 V 1133
1485 a(T)-7 b(ype)463 b(T)-7 b(ype)463 b(T)-7 b(ype)p
3248 1515 V 509 1614 V 899 1614 V 1037 1584 a(1)272 b(2)g(1)h(2)f(1)g
(2)185 b(Percentage)p 3248 1614 V 511 1618 2739 4 v 509
1717 4 100 v 561 1687 a(Estimate)p 899 1717 V 1213 1717
V 1528 1717 V 1842 1717 V 2156 1717 V 2470 1717 V 2784
1717 V 3248 1717 V 509 1817 V 671 1787 a(-1)p 899 1817
V 231 w(0.0\045)p 1213 1817 V 119 w(33.3\045)p 1528 1817
V 98 w(35.7\045)p 1842 1817 V 98 w(63.6\045)p 2156 1817
V 98 w(71.4\045)p 2470 1817 V 98 w(80.0\045)p 2784 1817
V 173 w(41.6\045)p 3248 1817 V 509 1917 V 685 1887 a(0)p
899 1917 V 224 w(21.7\045)p 1213 1917 V 98 w(33.3\045)p
1528 1917 V 98 w(35.7\045)p 1842 1917 V 98 w(36.4\045)p
2156 1917 V 98 w(19.0\045)p 2470 1917 V 98 w(20.0\045)p
2784 1917 V 173 w(26.0\045)p 3248 1917 V 509 2016 V 685
1986 a(1)p 899 2016 V 224 w(78.3\045)p 1213 2016 V 98
w(33.3\045)p 1528 2016 V 98 w(28.6\045)p 1842 2016 V
119 w(0.0\045)p 2156 2016 V 140 w(9.5\045)p 2470 2016
V 140 w(0.0\045)p 2784 2016 V 194 w(32.5\045)p 3248 2016
V 511 2020 2739 4 v 509 2119 4 100 v 622 2089 a(T)-7
b(otal)p 899 2119 V 160 w(29.9\045)p 1213 2119 V 119
w(3.9\045)p 1528 2119 V 119 w(18.2\045)p 1842 2119 V
98 w(14.3\045)p 2156 2119 V 98 w(27.3\045)p 2470 2119
V 119 w(6.5\045)p 2784 2119 V 173 w(100.0\045)p 3248
2119 V 511 2122 2739 4 v 933 2274 a Fe(T)g(ab)o(le)23
b(21.)h(Pr)n(opor)r(tions)e(of)h Fk(Size)g Fe(and)f Fk(T)-6
b(ype)23 b Fe(f)n(or)g Fk(Estimate)-83 2624 y Fj(Second,)34
b(again)d(common)f(to)j(both)e(v)n(ariables,)j(the)e(associa-)-182
2723 y(tions)15 b(using)g Fk(Size)g Fj(and)g Fk(T)-6
b(ype)15 b Fj(are)g(more)f(useful)h(from)f(both)h(research)-182
2823 y(and)24 b(practice)h(perspecti)n(v)o(es)f(in)i(that)f(the)o(y)g
(are)g(more)f(generically)-182 2923 y(applicable)f(and)h(acceptable)g
(when)g(compared)f(to)i(the)g(more)f(ob-)-182 3022 y(scure)c
Fk(Entry)g Fj(and)f Fk(Links)i Fj(measures.)-83 3122
y(Third,)i Fk(Links)h Fj(w)o(as)h(only)d(signi\002cant)h(for)g
(predicting)f(errors)h(\226)-182 3222 y(not)31 b(errors)f(of)h(an)o(y)g
(direction,)i(and)d Fk(Entry)i Fj(w)o(as)g(only)f(added)f(to)-182
3321 y(the)c(model)g(for)g(o)o(v)o(erestimates)f(as)i(the)g(third)f(v)n
(ariable,)h(suggest-)-182 3421 y(ing)c(that)g(the)g(majority)g(of)g
(the)g(association)g(is)h(contained)e(in)i Fk(Size)-182
3520 y Fj(and)19 b Fk(T)-6 b(ype)p Fj(.)-83 3620 y(The)25
b(o)o(v)o(erall)e(frequenc)o(y)f(for)i(the)h(reduced)e(set)i(of)f
Fk(Size)p Fj(,)i Fk(T)-6 b(ype)p Fj(,)-182 3720 y(and)30
b Fk(Estimate)h Fj(is)g(sho)n(wn)f(in)h(T)-7 b(ables)31
b(20)f(and)h(21.)56 b(The)30 b(full)h(ta-)-182 3819 y(ble)24
b(sho)n(wing)f(all)i(combinations)e(of)h(all)h(v)n(ariables)f(is)h
(less)g(useful)-182 3919 y(since)d(there)g(are)g(144)f(le)n(v)o(els)i
(of)f(the)g(six)h(v)n(ariables,)e(with)i(only)e(77)-182
4019 y(observ)n(ations.)44 b(In)27 b(addition)g(man)o(y)f(of)h(the)g
(observ)n(ations)f(f)o(all)i(in)-182 4118 y(the)20 b(same)g(cells,)h
(leading)e(to)h(lar)o(ge)g(numbers)e(of)i(empty)f(cells.)-83
4218 y(Ho)n(we)n(v)o(er)m(,)f(e)n(v)o(en)g(from)g(this)i(simple)f
(presentation)f(it)i(again)e(ap-)-182 4317 y(pears)25
b(that)i(there)e(is)i(some)f(relationship)f(between)g
Fk(Size)h Fj(and)g Fk(Es-)-182 4417 y(timate)g Fj(and)f(between)h
Fk(T)-6 b(ype)26 b Fj(and)f Fk(Estimate)i Fj(e)n(v)o(en)e(when)g(the)i
(tw)o(o)-182 4517 y(v)n(ariables)15 b(are)i(included)e(\(as)i(compared)
e(to)h(the)h(pairwise)f(compar)n(-)-182 4616 y(isons)k(in)g(Section)g
(5)g(on)g(contingenc)o(y)d(table)j(analysis\).)25 b(Ov)o(eresti-)-182
4716 y(mates)20 b(are)g(more)g(common)e(for)h(small)i(modules)e(and)h
(reports.)-182 4921 y Fd(7.2)90 b(Analysis)-83 5126 y
Fj(T)-7 b(able)24 b(22)g(sho)n(ws)g(the)h(results)f(of)g(the)g(v)n
(arious)g(models)f(de)n(v)o(el-)-182 5225 y(oped.)58
b(From)31 b(T)-7 b(able)31 b(23)h(the)f(signi\002cance)g(of)g(the)h
(interactions)-182 5325 y(between)19 b Fk(T)-6 b(ype)19
b Fj(and)h Fk(Estimate)g Fj(as)g(well)h(as)f(between)f
Fk(Size)h Fj(and)g Fk(Es-)1974 2624 y(timate)k Fj(can)g(be)g(seen.)38
b(Ho)n(we)n(v)o(er)22 b(the)j(full)f(model)f(with)h(the)h(three)1974
2723 y(w)o(ay)32 b(interactions)e(sho)n(ws)i(that)g(the)g(interaction)e
(between)h Fk(Size)p Fj(,)1974 2823 y Fk(T)-6 b(ype)p
Fj(,)20 b(and)f Fk(Estimate)i Fj(is)g(not)f(signi\002cant.)2073
2934 y(All)h(parameters)e(are)i(signi\002cant)e(e)o(xcept)h(for)f
(interactions)h(be-)1974 3034 y(tween)g Fk(T)-6 b(ype)p
Fj(=1)19 b(and)g Fk(Size)p Fj(=1,)h Fk(Estimate)p Fj(=-1)f(and)g
Fk(Size)p Fj(=2,)h(and)f Fk(Es-)1974 3134 y(timate)p
Fj(=0)25 b(and)g Fk(Size)p Fj(=2.)42 b(The)26 b(other)f(11)g
(parameters)g(are)h(all)g(sig-)1974 3233 y(ni\002cant.)46
b(Residuals)28 b(are)f(normally)f(distrib)n(uted)h(and)g(appear)f(to)
1974 3333 y(be)20 b(independent.)2073 3444 y(The)25 b(e)o(xpected)e
(counts)h(are)g(sho)n(wn)g(in)h(T)-7 b(able)24 b(24.)38
b(From)24 b(here)1974 3544 y(the)d(odds)g(of)g(a)g(correct)g(estimate)g
(for)g(a)g(small)h(screen)f(are)g(0.22:1,)1974 3644 y(for)g(a)h(small)g
(report)f(the)o(y)g(are)g(1.47:1)g(gi)n(ving)f(a)i(log-odds)e(ratio)h
(of)1974 3743 y(6.7)27 b(in)h(f)o(a)n(v)n(or)g(of)g(the)g(reports.)47
b(F)o(or)28 b(medium)f(modules)g(the)h(cor)n(-)1974 3843
y(responding)20 b(odds)h(are)i(0.62:1)d(and)i(0.49:1)f(respecti)n(v)o
(ely)f(and)i(the)1974 3942 y(ratio)d(is)i(1.27)d(in)i(f)o(a)n(v)n(or)f
(of)h(screens.)k(F)o(or)c(lar)o(ge)e(modules)h(the)h(odds)1974
4042 y(are)f(0.27:1)g(and)g(0.13:1)f(respecti)n(v)o(ely)g(with)i(a)g
(ratio)f(of)h(2.08)e(in)i(f)o(a-)1974 4142 y(v)n(or)g(of)h(screens.)28
b(F)o(or)20 b(all)i Fk(T)-6 b(ype)21 b Fj(and)f Fk(Size)i
Fj(combinations)d(the)i(mis-)1974 4241 y(estimates)j(outnumber)c(the)k
(correct)e(estimates)i(e)o(xcept)e(for)h(small)1974 4341
y(reports.)35 b(As)25 b(the)f(modules)f(get)h(lar)o(ger)f(the)h(ratio)g
(becomes)f(more)1974 4441 y(f)o(a)n(v)n(orable)18 b(to)i(screens,)g
(suggesting)e(that)i(ef)n(fort)e(for)i(lar)o(ge)e(screens)1974
4540 y(is)j(easier)f(to)h(estimate)f(than)g(for)f(lar)o(ge)h(reports.)
1974 4783 y Fd(7.3)90 b(Conclusions)21 b(fr)n(om)i(log\255linear)f
(analysis)2073 5026 y Fj(From)17 b(this)h(basic)g(analysis)f(it)h(can)f
(be)g(seen)h(that)f(both)g Fk(T)-6 b(ype)17 b Fj(and)1974
5126 y Fk(Size)j Fj(of)n(fer)e(considerable)g(potential)h(in)g(being)g
(able)h(to)f(predict)g(the)1974 5225 y(probability)d(of)h
(misestimation.)24 b(Modules)17 b(are)h(more)f(lik)o(ely)h(to)g(be)1974
5325 y(underestimated)24 b(if)i(the)o(y)g(are)g(screens)g(rather)f
(than)h(reports,)h(and)1839 5574 y(10)p eop
%%Page: 11 11
11 10 bop 99 3 3563 4 v 97 103 4 100 v 149 73 a Fj(Model)p
2556 103 V 2240 w(De)n(viance)p 2967 103 V 112 w(df)p
3149 103 V 99 w(Signi\002cance)p 3659 103 V 99 106 3563
4 v 97 206 4 100 v 149 176 a(T)-7 b(ype)20 b(+)g(Size)h(+)f(Estimate)p
2556 206 V 1726 w(60.5808)p 2967 206 V 96 w(12)p 3149
206 V 281 w(0.0000)p 3659 206 V 97 306 V 149 276 a(T)-7
b(ype)20 b(+)g(Size)h(+)f(Estimate)h(+)f(T)-7 b(ype*Size)p
2556 306 V 1288 w(52.8846)p 2967 306 V 96 w(10)p 3149
306 V 281 w(0.0000)p 3659 306 V 97 405 V 149 375 a(T)g(ype)20
b(+)g(Size)h(+)f(Estimate)h(+)f(T)-7 b(ype*Size)20 b(+)g(T)-7
b(ype*Estimate)p 2556 405 V 703 w(42.0105)p 2967 405
V 138 w(8)p 3149 405 V 281 w(0.0000)p 3659 405 V 97 505
V 149 475 a(T)g(ype)20 b(+)g(Size)h(+)f(Estimate)h(+)f(T)-7
b(ype*Size)20 b(+)g(Size*Estimate)p 2556 505 V 725 w(11.8810)p
2967 505 V 138 w(6)p 3149 505 V 281 w(0.0647)p 3659 505
V 97 604 V 149 575 a(T)-7 b(ype)20 b(+)g(Size)h(+)f(Estimate)h(+)f(T)-7
b(ype*Size)20 b(+)g(T)-7 b(ype*Estimate)19 b(+)i(Size*Estimate)p
2556 604 V 223 w(3.032)p 2967 604 V 139 w(4)p 3149 604
V 281 w(0.5539)p 3659 604 V 97 704 V 149 674 a(Full)f(model)p
2556 704 V 2365 w(0)p 2967 704 V 141 w(0)p 3149 704 V
482 w(-)p 3659 704 V 99 707 3563 4 v 121 859 a Fe(T)-7
b(ab)o(le)24 b(22.)f(The)g(results)g(of)g(the)f(log\255linear)h(anal)o
(ysis)g(using)g Fk(Size)g Fe(and)f Fk(T)-6 b(ype)23 b
Fe(as)g(predictor)o(s)g(of)f Fk(Estimate)p 728 1013 2305
4 v 726 1113 4 100 v 778 1083 a Fj(T)-6 b(erms)p 1567
1113 V 634 w(Change)19 b(in)i(De)n(viance)p 2338 1113
V 111 w(df)p 2521 1113 V 99 w(Signi\002cance)p 3031 1113
V 728 1116 2305 4 v 726 1216 4 100 v 778 1186 a(All)f(interactions)p
1567 1216 V 727 w(60.5808)p 2338 1216 V 97 w(12)p 2521
1216 V 280 w(0.0000)p 3031 1216 V 726 1315 V 778 1286
a(All)g(2-w)o(ay)g(interactions)p 1567 1315 V 541 w(57.548)p
2338 1315 V 138 w(8)p 2521 1315 V 281 w(0.0000)p 3031
1315 V 726 1415 V 778 1385 a(T)-7 b(ype*Estimate)p 1567
1415 V 827 w(8.849)p 2338 1415 V 139 w(2)p 2521 1415
V 281 w(0.0117)p 3031 1415 V 726 1515 V 778 1485 a(Size*Estimate)p
1567 1515 V 766 w(38.9785)p 2338 1515 V 138 w(2)p 2521
1515 V 281 w(0.0000)p 3031 1515 V 726 1614 V 778 1584
a(T)g(ype*Estimate*Size)p 1567 1614 V 642 w(3.032)p 2338
1614 V 139 w(4)p 2521 1614 V 281 w(0.5539)p 3031 1614
V 728 1618 2305 4 v -83 1770 a Fe(T)g(ab)o(le)35 b(23.)h(The)e(anal)o
(ysis)i(of)e(de)o(viance)h(of)f(the)g(log\255linear)h(anal)o(ysis)g
(using)g Fk(Size)f Fe(and)h Fk(T)-6 b(ype)34 b Fe(as)h(predictor)o(s)g
(of)-83 1869 y Fk(Estimate)p 775 2007 2211 4 v 773 2106
4 100 v 1163 2106 V 1870 2076 a Fj(Size)p 2984 2106 V
773 2206 V 1163 2206 V 1410 2176 a(1)468 b(2)h(3)p 2984
2206 V 773 2305 V 1163 2305 V 1349 2276 a(T)-7 b(ype)345
b(T)-7 b(ype)346 b(T)-7 b(ype)p 2984 2305 V 773 2405
V 1163 2405 V 1287 2375 a(1)224 b(2)203 b(1)g(2)223 b(1)h(2)151
b(T)-7 b(otal)p 2984 2405 V 775 2408 2211 4 v 773 2508
4 100 v 824 2478 a(Estimate)p 1163 2508 V 1450 2508 V
1695 2508 V 1939 2508 V 2184 2508 V 2471 2508 V 2716
2508 V 2984 2508 V 773 2608 V 935 2578 a(-1)p 1163 2608
V 230 w(0.52)p 1450 2608 V 119 w(0.48)p 1695 2608 V 98
w(4.86)p 1939 2608 V 98 w(7.14)p 2184 2608 V 98 w(14.63)p
2471 2608 V 97 w(4.37)p 2716 2608 V 141 w(32)p 2984 2608
V 773 2707 V 949 2677 a(0)p 1163 2707 V 244 w(4.22)p
1450 2707 V 119 w(1.78)p 1695 2707 V 98 w(5.38)p 1939
2707 V 98 w(3.62)p 2184 2707 V 119 w(4.40)p 2471 2707
V 118 w(0.60)p 2716 2707 V 141 w(20)p 2984 2707 V 773
2807 V 949 2777 a(1)p 1163 2807 V 224 w(18.27)p 1450
2807 V 97 w(0.73)p 1695 2807 V 98 w(3.76)p 1939 2807
V 98 w(0.24)p 2184 2807 V 119 w(1.97)p 2471 2807 V 118
w(0.03)p 2716 2807 V 141 w(25)p 2984 2807 V 775 2810
2211 4 v 773 2910 4 100 v 885 2880 a(T)g(otal)p 1163
2910 V 213 w(23)p 1450 2910 V 202 w(3)p 1695 2910 V 182
w(14)p 1939 2910 V 161 w(11)p 2184 2910 V 182 w(21)p
2471 2910 V 202 w(5)p 2716 2910 V 194 w(77)p 2984 2910
V 775 2913 2211 4 v 712 3065 a Fe(T)g(ab)o(le)24 b(24.)f(Expected)g
(frequenc)o(y)g(of)g Fk(Size)p Fe(,)g Fk(T)-6 b(ype)p
Fe(,)22 b Fk(Estimate)h Fe(v)n(ariab)o(les)-182 3415
y Fj(are)30 b(lar)o(ger)f(rather)h(than)g(smaller)-5
b(.)56 b(The)31 b(opposite)e(relationships)-182 3515
y(apply)19 b(for)g(o)o(v)o(erestimates.)-182 3904 y Fl(8)99
b(Estimate)25 b(accuracy)h(as)e(a)h(function)h(of)f(time)-83
4276 y Fj(One)d(\002nal)h(analysis)f(procedure)e(w)o(as)j(undertak)o
(en)d(in)j(order)e(to)-182 4376 y(determine)15 b(whether)h(estimate)i
(accurac)o(y)d(might)h(change)g(o)o(v)o(er)g(the)-182
4476 y(period)30 b(of)i(the)g(project.)60 b(In)32 b(terms)g(of)g(this)g
(particular)f(system,)-182 4575 y(reasonable)19 b(accurac)o(y)f(w)o(as)
k(achie)n(v)o(ed)c(at)j(the)g(be)o(ginning)d(and)h(end)-182
4675 y(of)24 b(the)h(project,)g(whilst)g(the)g(most)g(important)f
(errors)g(\(underesti-)-182 4775 y(mates\))c(occurred)e(in)i(the)g
(middle)g(of)g(the)g(project.)-83 4926 y(This)37 b(suggests)f(that)g
(either)g(ef)n(fort)g(for)f(the)i(be)o(ginning)c(and)-182
5026 y(end)21 b(tasks)h(w)o(as)h(simply)e(easier)h(to)g(predict)f
(correctly)-5 b(,)21 b(or)g(perhaps)-182 5126 y(the)h(estimates)i(were)
e(re)n(visited)h(to)n(w)o(ards)f(the)h(end)f(of)g(the)h(project,)-182
5225 y(meaning)g(that)i(tasks)g(occurring)e(at)i(or)f(after)h(that)g
(point)f(were)g(no)-182 5325 y(longer)18 b(subject)i(to)h
(signi\002cant)e(ef)n(fort)h(error)-5 b(.)1974 3415 y
Fl(9)99 b(Conclusions)2073 3631 y Fj(After)31 b(the)g(analysis)g(of)g
(the)g(dataset)g(using)f(contingenc)o(y)e(ta-)1974 3731
y(ble)f(analysis,)h(logistic)e(re)o(gression)f(and)i(log-linear)e
(modeling)f(it)1974 3830 y(appears)d(that)h(for)f(this)i(particular)e
(dataset)h(there)f(are)h(indeed)f(sys-)1974 3930 y(tematic)f(biases)i
(in)e(the)h(estimates)g(of)g(ef)n(fort.)k(These)20 b(biases)h(e)o
(xist,)1974 4030 y(at)d(least,)h(for)f(the)g(size)g(and)g(type)f(of)h
(the)g(modules)f(and)g(also)i(for)e(the)1974 4129 y(e)o(xistence)k(of)g
(links)h(to)f(multiple)g(tables)h(and)f(data)g(entry)g(capabil-)1974
4229 y(ity)-5 b(.)24 b(These)18 b(latter)g(tw)o(o)h(characteristics)f
(are)g(ho)n(we)n(v)o(er)e(also)j(related)1974 4329 y(to)h(the)g(size)h
(and)f(type)g(of)g(module.)2073 4428 y(It)c(is)g(of)f(course)g
(possible)g(to)g(produce)e(models)i(that)g(adjust)g(their)1974
4528 y(estimates)32 b(for)e(these)h(types)g(of)g(modules)f(while)h
(using)g(the)g(full)1974 4628 y(set)23 b(of)f(a)n(v)n(ailable)g
(numerical)f(data)h(for)g(ef)n(fort)f(estimates.)32 b(Simply)1974
4727 y(by)26 b(including)e(binary)h(v)n(ariables)g(in)h(such)g(models)g
(the)g(intercept)1974 4827 y(w)o(ould)f(be)g(changed)f(to)i(re\003ect)g
(dif)n(ferences.)39 b(Alternati)n(v)o(ely)-5 b(,)25 b(for)1974
4926 y(v)n(ariables)i(such)h(as)g(size)h(modi\002cation)d(to)i(the)g
(slope)g(parameter)1974 5026 y(may)20 b(be)g(made.)2073
5126 y(Also,)28 b(the)f(results)f(could)g(be)g(used)g(in)h(educating)d
(project)i(es-)1974 5225 y(timators)d(as)h(to)g(where)f(their)g
(estimates)h(are)f(going)g(wrong.)33 b(This)1974 5325
y(w)o(ould)26 b(pro)o(vide)f(a)i(useful)f(feedback)f(mechanism,)i
(possibly)f(ac-)1839 5574 y(11)p eop
%%Page: 12 12
12 11 bop -182 83 a Fj(celerating)19 b(the)h(learning)f(process)g(for)h
(project)f(managers.)-83 194 y(Similarly)-5 b(,)24 b(models)f(of)g
(risk)g(could)g(be)g(constructed)f(for)h(those)-182 294
y(modules)31 b(that)h(are)g(more)g(lik)o(ely)g(to)g(be)g(misestimated,)
j(includ-)-182 393 y(ing)29 b(some)h(accounting)e(for)h(the)i(lik)o
(ely)e(magnitude)g(\(more)f(than)-182 493 y(the)g(three)g(le)n(v)o(els)
h(used)f(here)g(could)g(allo)n(w)g(for)g(this\))h(and)f(direc-)-182
592 y(tion)16 b(\(which)h(has)g(been)g(sho)n(wn)f(here)h(to)g(be)g
(associated)g(with)g(some)-182 692 y(module)h(characteristics\).)-83
803 y(A)k(number)c(of)j(limitations)g(on)f(this)h(analysis)g(need)f(to)
h(be)g(con-)-182 903 y(sidered)g(when)g(e)o(xamining)e(the)j(results.)
30 b(First,)22 b(the)g(ob)o(vious)e(lim-)-182 1002 y(itation)f(of)g(ha)
n(ving)g(only)g(the)g(one)g(dataset)h(is)g(that)g(there)f(is)i(no)e(w)o
(ay)-182 1102 y(of)32 b(kno)n(wing)f(if)j(the)f(biases)g(were)g(due)f
(to)i(some)e(special)h(char)n(-)-182 1202 y(acteristics)e(of)g(the)h
(project)e(or)h(if)h(the)f(manager)f(mak)o(es)h(similar)-182
1301 y(misjudgements)d(on)i(their)g(other)g(projects.)55
b(Second,)31 b(there)f(are)-182 1401 y(se)n(v)o(eral)23
b(other)h(v)n(ariables)g(that)g(could)g(also)g(ha)n(v)o(e)g(been)g
(related)g(to)-182 1500 y(estimate)c(accurac)o(y)f(that)h(were)g(not)g
(able)g(to)g(be)g(considered.)-83 1611 y(Despite)i(these)f(objections,)
g(the)g(analysis)h(does)f(at)h(least)g(sho)n(w)-182 1711
y(that)j(for)g(this)h(particular)e(project)g(manager)g(and)h(system)g
(combi-)-182 1811 y(nation)20 b(there)h(were)h(systematic)f(biases)i
(on)e(the)g(ef)n(fort)g(prediction)-182 1910 y(in)m(v)n(olving)i(size)k
(and)e(type)h(of)f(the)h(constituent)f(modules.)41 b(These)-182
2010 y(tw)o(o)28 b(f)o(actors)f(are)h(perhaps)f(the)h(most)g(intuiti)n
(v)o(e)f(and)h(reasonable,)-182 2110 y(pro)o(viding)20
b(a)k(foundation)c(for)j(further)e(analysis)i(into)g(other)g(asso-)-182
2209 y(ciations)j(for)h(the)g(purposes)f(of)g(de)n(v)o(eloping)f
(correcti)n(v)o(e)g(models,)-182 2309 y(educating)19
b(and)i(assessing)h(project)f(managers,)f(and)h(making)f(risk)-182
2408 y(estimates.)-182 2675 y Fl(Refer)n(ences)-145 2916
y Fb([1])42 b(A.)29 b(Albrecht)g(and)i(J.)e(G.)g(Jr)l(.)61
b(Softw)o(are)29 b(function,)k(source)d(lines)-16 3008
y(of)24 b(code,)i(and)f(de)n(v)o(elopment)h(ef)n(fort)f(prediction:)34
b(a)25 b(softw)o(are)f(sci-)-16 3099 y(ence)h(v)n(alidation.)46
b Fa(IEEE)24 b(T)l(r)o(ansactions)h(on)h(Softwar)m(e)f(Engineer)o(-)-16
3190 y(ing)p Fb(,)18 b(9\(6\):639\226648,)k(1983.)-145
3337 y([2])42 b(B.)17 b(Boehm.)26 b Fa(Softwar)m(e)18
b(Engineering)i(Economics)p Fb(.)25 b(Prentice-Hall,)-16
3429 y(Engle)n(w)o(ood)20 b(Clif)n(fs)e(NJ,)g(USA,)g(1981.)2011
83 y([3])42 b(A.)24 b(Gray)h(and)h(S.)e(MacDonell.)48
b(Fuzzy)25 b(logic)g(for)g(softw)o(are)h(met-)2140 174
y(ric)g(models)h(throughout)h(the)f(de)n(v)o(elopment)h(life)e(c)o
(ycle.)51 b(In)27 b Fa(Pr)m(o-)2140 266 y(ceedings)20
b(of)f(the)g(1999)h(Annual)f(Meeting)g(of)g(the)g(North)g(American)2140
357 y(Fuzzy)j(Information)h(Pr)m(ocessing)f(Society)g(-)g(N)n(AFIPS)p
Fb(,)e(Ne)n(w)i(Y)-8 b(ork)2140 448 y(NY)e(,)18 b(USA,)g(1999.)i(IEEE)e
(Computer)h(Society)g(Press.)26 b(T)-6 b(o)19 b(appear)l(.)2011
573 y([4])42 b(F)-6 b(.)19 b(Heemstra)h(and)h(R.)f(K)o(usters.)30
b(Function)21 b(point)f(analysis:)27 b(e)n(v)n(alu-)2140
664 y(ation)17 b(of)h(a)f(softw)o(are)g(cost)h(estimation)f(model.)24
b Fa(Eur)m(opean)18 b(J)n(ournal)2140 756 y(of)h(Information)g(Systems)
p Fb(,)g(1\(4\):229\226237,)i(1991.)2011 880 y([5])42
b(M.)16 b(Host)f(and)i(C.)e(W)-6 b(ohlin.)20 b(A)c(subjecti)n(v)o(e)h
(ef)n(fort)e(estimation)h(e)o(xper)o(-)2140 971 y(iment.)46
b Fa(Information)26 b(and)g(Softwar)m(e)g(T)-7 b(ec)o(hnolo)o(gy)p
Fb(,)28 b(39:755\226762,)2140 1063 y(1997.)2011 1187
y([6])42 b(R.)21 b(Hughes.)36 b(Expert)21 b(judgement)i(as)f(an)g
(estimating)f(method.)36 b Fa(In-)2140 1279 y(formation)19
b(and)h(Softwar)m(e)g(T)-7 b(ec)o(hnolo)o(gy)p Fb(,)20
b(38:67\22675,)h(1996.)2011 1403 y([7])42 b(W)-7 b(.)21
b(Humphre)o(y)-5 b(.)37 b Fa(A)22 b(Discipline)g(for)f(Softwar)m(e)i
(Engineering)p Fb(.)37 b(Ad-)2140 1494 y(dison)19 b(W)-6
b(esle)o(y)h(,)19 b(1995.)2011 1619 y([8])42 b(A.)35
b(Lederer)g(and)h(J.)f(Prasad.)79 b(Nine)36 b(management)g(guidelines)
2140 1710 y(for)c(better)h(cost)g(estimating.)70 b Fa(Communications)34
b(of)f(the)f(A)n(CM)p Fb(,)2140 1802 y(35\(2\):51\22659,)21
b(1992.)2011 1926 y([9])42 b(A.)21 b(Lederer)h(and)h(J.)f(Prasad.)36
b(A)22 b(causal)g(model)h(for)f(softw)o(are)g(cost)2140
2017 y(estimating)g(error)l(.)39 b Fa(IEEE)21 b(T)l(r)o(ansactions)i
(on)g(Softwar)m(e)g(Engineer)o(-)2140 2109 y(ing)p Fb(,)c
(24\(2\):137\226148,)i(1998.)1974 2233 y([10])42 b(R.)18
b(Meli.)27 b(Early)19 b(function)g(points:)24 b(a)19
b(ne)n(w)g(estimation)h(method)f(for)2140 2325 y(softw)o(are)e
(projects.)24 b(In)17 b Fa(Pr)m(occeedings)i(ESCOM97)p
Fb(,)f(Berlin,)e(1997.)1974 2449 y([11])42 b(T)-6 b(.)15
b(Mukhopadhyay)-5 b(,)19 b(S.)c(V)l(icinanza,)h(and)g(M.)f(Prietula.)k
(Examining)2140 2540 y(the)j(feasibility)f(of)h(a)g(case-based)h
(reasoning)g(model)f(for)g(softw)o(are)2140 2632 y(ef)n(fort)16
b(estimation.)k Fa(MIS)c(Quarterly)p Fb(,)h(pages)f(155\226171,)j(June)
d(1992.)1974 2756 y([12])42 b(M.)22 b(Shepperd)h(and)g(C.)f
(Scho\002eld.)38 b(Estimating)22 b(softw)o(are)g(project)2140
2848 y(ef)n(fort)17 b(using)h(analogies.)24 b Fa(IEEE)16
b(T)l(r)o(ansactions)i(on)g(Softwar)m(e)g(Engi-)2140
2939 y(neering)p Fb(,)h(23\(12\):736\226743,)j(1997.)1974
3064 y([13])42 b(E.)18 b(Stensrud)h(and)g(I.)f(Myrtv)o(eit.)27
b(Human)19 b(performance)h(estimating)2140 3155 y(with)30
b(analogy)h(and)g(re)o(gression)f(models:)47 b(an)30
b(empirical)g(v)n(alida-)2140 3246 y(tion.)24 b(In)17
b Fa(Pr)m(oceedings)i(of)e(the)h(F)m(ifth)e(International)j(Softwar)m
(e)g(Met-)2140 3337 y(rics)d(Symposium)i(\(Metrics'98\))p
Fb(,)f(pages)h(205\226213,)h(Los)d(Alamitos,)2140 3429
y(California,)i(1998.)i(IEEE)e(Computer)h(Society)g(Press.)1839
5574 y Fj(12)p eop
%%Trailer
end
userdict /end-hook known{end-hook}if
%%EOF