!g@p  dZddlmZddlZdZdZdZd3dZd3dZ d4d Z d5d Z d6d Z d Z d6dZGddZdZGddZGddZedk(r0eZej+ej-ej/ddlmZgddZgdZddgZe eeZeeej@dZ!ej@ddzZ"ej@ddzdzZ#devre e"\Z$Z%e e#\Z&Z'd evr*ejPe"d!d"Z"ejPe#d!d"Z#e"jRdZ*e e"\Z$Z%e e#\Z&Z'e e$e&d #Z+e e$e&d$#Z,e e$e&d%#Z-eej\e+j^e+ej`e+jcdk(jed&d'gdZ3d(d)gdZ4e4ejjjme*zZ7e3ej@dd*zZ8e+e8zjcde7zZ9e,e8zjcde7zZ:e-e8zjcde7zZ;ejxe9e+j{Z>ejxe9e-j{Z?e>jZAe?jZBejxe;e+j{ZCejxe;e-j{ZDeCjZEeDjZFejje-e+d+,dZIejje+e-d+,dZJeej\eJeFeEz eej\eIeAeBz ee+e-ZKeejeje+eKje-z eejeje-eKje+z eejejeFeKjeEz ejejeBeKjeAz e ee\ZQZRZSZTZUZVed-ed.eeSeeCjeRjed/ed0eeUeeCjeTjeCjeTejejd1ejdfZ]e?jd2dZ_e>jeKje]jZ`eejeje_e`z ee;e"e#Zaeeajjeeajjeeajjeeajjeeajddjeeajddjeeaj y)7afunctions to work with contrasts for multiple tests contrast matrices for comparing all pairs, all levels to reference level, ... extension to 2-way groups in progress TwoWay: class for bringing two-way analysis together and try out various helper functions Idea for second part - get all transformation matrices to move in between different full rank parameterizations - standardize to one parameterization to get all interesting effects. - multivariate normal distribution - exploit or expand what we have in LikelihoodResults, cov_params, f_test, t_test, example: resols_dropf_full.cov_params(C2) - connect to new multiple comparison for contrast matrices, based on multivariate normal or t distribution (Hothorn, Bretz, Westfall) ) assert_equalNcg}t|D]F}t|dz|D]2}tj|}d||<d||<|j|4Htj|S)zcontrast or restriction matrix for all pairs of nm variables Parameters ---------- nm : int Returns ------- contr : ndarray, 2d, (nm*(nm-1)/2, nm) contrast matrix for all pairwise comparisons )rangenpzerosappendarray)nmcontrij contr_rows e/var/www/dash_apps/app1/venv/lib/python3.12/site-packages/statsmodels/sandbox/stats/contrast_tools.pycontrast_allpairsrsp E 2Y$qsB $A IIaLIaL LL #  $$ 88E?ctjtj|dz tj|dz  f}|S)zcontrast or restriction matrix for all against first comparison Parameters ---------- nm : int Returns ------- contr : ndarray, 2d, (nm-1, nm) contrast matrix for all against first comparisons r)r column_stackoneseye)r r s rcontrast_all_oner5s6 OORWWRT]RVVBqD\M: ;E Lrcbtj|tj||f|z z S)zcontrast or restriction matrix for all against mean comparison Parameters ---------- nm : int Returns ------- contr : ndarray, 2d, (nm-1, nm) contrast matrix for all against mean comparisons )rrr)r s rcontrast_diff_meanrEs) 66":B(+ ++rc|dvr:|stj|dk\rdSdStj|dk\rdSdSt|S)N)rrrr+-)rsignstr)xnopluss rsignstrr#TsGH}''!*a-3 0S 0Q2 /C /1v rc|rtddd}n td}|Dcgc]G}djt|||Dcgc]\}}|dk7rt|d|c}}I}}}}|Scc}}wcc}}}w)NrrrTr")slicejoinzipr#) contrastsnamesreverseslrowcvlabelss rcontrast_labelsr1^s 4r " 4[)233!$gg%(e_R%8D!aAF"!D121#6DE3F3 MD3s"A: A4$ A:4A:ct|}t|}|Dcgc]}|D] }|d| } }}tjd|f} d| d<|s tj| t | f} ntj| t | f} tj | ddtj|} t| | d} | Dcgc]J}djt|| dddDcgc]\}}|d k7rt|d |c}}L}}}}tjd|f}d|d<|s tj|t | f}ntj|t | f}tj tj||dd}|Dcgc]J}djt|| dddDcgc]\}}|d k7rt|d |c}}L}}}}|+|)t|\}}t|\}}t||}nd}| | ||||fScc}}wcc}}wcc}}}wcc}}wcc}}}w) abuild contrast matrices for products of two categorical variables this is an experimental script and should be converted to a class Parameters ---------- names1, names2 : lists of strings contains the list of level labels for each categorical variable intgroup1, intgroup2 : ndarrays TODO: this part not tested, finished yet categorical variable Notes ----- This creates a full rank matrix. It does not do all pairwise comparisons, parameterization is using contrast_all_one to get differences with first level. ? does contrast_all_pairs work as a plugin to get all pairs ? _r)rrNT)r+rrrr%)lenrr r_rrkronrr1r'r(r#dummy_1d dummy_product)names1names2 intgroup1 intgroup2pairsn1n2rr names_prodee1dd contrast_prodnames_contrast_prod0r-r.r/names_contrast_prodee2dd2contrast_prod2names_contrast_prod2d1r3d2dummys rcontrast_productrMhs. VB VB&,=f=QCq*=*=J= ((Ab6 CCH  UU3)"--- . UU3*2... /GGBqrFBFF2J/M*=*dS-:;;%(77),S*)=dd)C$O"%!AqAv(/q'>&?s$C$OP;; ((Ab6 CCH eeC*2.../eeC+B///0WWRVVBZQR1N-;<<%(GG),S*)=dd)C%O"%!AqAv)0$(?'@%D%OP<< I$9#A#Ab"%  ':&($O;%O>> x = np.array(['F', 'F', 'M', 'M', 'F', 'F', 'M', 'M', 'F', 'F', 'M', 'M'], dtype='|S1') >>> dummy_1d(x, varname='gender') (array([[1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1]]), ['gender_F', 'gender_M']) Nrzlevel_%dz_%s)rmaxrarangeastypeintuniquer )r!varnamerr0 grouplabelss rr7r7s\*/! *<=Q*q.==!D& 299QUUWQY//77As C C fullcf|dk(r8|dddddf|dddddfzj|jdd}|S|dk(rpt|ddddf|ddddf}tjtj |jdt |ddddf|ddddf|f}|S|dk(rpt|ddddf|ddddf}tjtj |jdt |ddddf|ddddf|f}|Std) adummy variable from product of two dummy variables Parameters ---------- d1, d2 : ndarray two dummy variables, assumes full set for methods 'drop-last' and 'drop-first' method : {'full', 'drop-last', 'drop-first'} 'full' returns the full product, encoding of intersection of categories. The drop methods provide a difference dummy encoding: (constant, main effects, interaction effects). The first or last columns of the dummy variable (i.e. levels) are dropped to get full rank dummy matrix. Returns ------- dummy : ndarray dummy variable for product, see method rVNrr drop-last drop-firstrmethod not recognized)reshapeshaper8rrrrR ValueError)rJrKmethodrBd12rld12rs rr8r8s:.1Tl2aQh<' 0 0!R @ I ; b3B3iAcrcE3 __bggbhhqk37AcrcEBq"uIeT U I < R!"Xr!AB$x0 __bggbhhqk37AabD2ad8DQ R I011rc.|j\}}tjtj|ddk(\}}tjtj|ddk(\}}tj|}tj dg|f|k(j r*tj ||dz gf|k(j s tdtj dg|dzf}tj |dz|gf} || fS)astart and endpoints of groups in a sorted dummy variable array helper function for nested categories Examples -------- >>> d1 = np.array([[1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1]]) >>> dummy_limits(d1) (array([0, 4, 8]), array([ 4, 8, 12])) get group slices from an array >>> [np.arange(d1.shape[0])[b:e] for b,e in zip(*dummy_limits(d1))] [array([0, 1, 2, 3]), array([4, 5, 6, 7]), array([ 8, 9, 10, 11])] >>> [np.arange(d1.shape[0])[b:e] for b,e in zip(*dummy_limits(d1))] [array([0, 1, 2, 3]), array([4, 5, 6, 7]), array([ 8, 9, 10, 11])] raxisrrzdummy variable is not sorted)r\rnonzerodiffrPr5allr]) dnobsnvarsstart1col1end1col1_ccstartends r dummy_limitsrqs:''KD%::bggaQ/23LFD**RWWQA.23KD% 5 BeeQCI"$ ) ) +UU557)+,2779788 EE1#vax- E %%Q C #:rc|dk(r|St|\}}t|\}}tj||}tj||}||k(} || z} || z} |dk(rlt|ddddf|ddddf} tjtj |j dt|ddddf|dd| ff} n||dk(rlt|ddddf|ddddf}tjtj |j dt|ddddf|dd| ff} n td| | | fS) aunfinished and incomplete mainly copy past dummy_product dummy variable from product of two dummy variables Parameters ---------- d1, d2 : ndarray two dummy variables, d2 is assumed to be nested in d1 Assumes full set for methods 'drop-last' and 'drop-first'. method : {'full', 'drop-last', 'drop-first'} 'full' returns the full product, which in this case is d2. The drop methods provide an effects encoding: (constant, main effects, subgroup effects). The first or last columns of the dummy variable (i.e. levels) are dropped to get full rank encoding. Returns ------- dummy : ndarray dummy variable for product, see method rVrXNrrrYrrZ) rqrin1dr8rrr\rRr])rJrKr^rjrlstart2end2firstlastequal col_dropf col_droplr_rBr`s r dummy_nestedr{2sU, #LFD#LFD GGFF #E 774 D d]Ev Iuf Ib3B3iAcrcE3 __bggbhhqk37AcrcEBq{OT U < R!"Xr!AB$x0 __bggbhhqk37AabD2a k?S T011 y) ##rc.eZdZdZdZdZdZdZdZy)DummyTransformaConversion between full rank dummy encodings y = X b + u b = C a a = C^{-1} b y = X C a + u define Z = X C, then y = Z a + u contrasts: R_b b = r R_a a = R_b C a = r where R_a = R_b C Here C is the transform matrix, with dot_left and dot_right as the main methods, and the same for the inverse transform matrix, C^{-1} Note: - The class was mainly written to keep left and right straight. - No checking is done. - not sure yet if method names make sense ctjj||dd|_tjj||dd|_y)z\C such that d1 C = d2, with d1 = X, d2 = Z should be (x, z) in arguments ? rrcondrN)rlinalglstsq transf_matrixinvtransf_matrix)selfrJrKs r__init__zDummyTransform.__init__sD  YY__R2_>qA " Bb A! DrcBtj|j|S)z b = C a rdotr)ras rdot_leftzDummyTransform.dot_leftsvvd((!,,rcBtj||jS)z z = x C r)rr!s r dot_rightzDummyTransform.dot_rightsvva++,,rcBtj|j|S)z a = C^{-1} b rrr)rbs r inv_dot_leftzDummyTransform.inv_dot_leftsvvd++Q//rcBtj||jS)z x = z C^{-1} r)rzs r inv_dot_rightzDummyTransform.inv_dot_rightsvva..//rN) __name__ __module__ __qualname____doc__rrrrrrrr}r}as"@E- - 0 0rr}ctj|}|jdz}tdgdg|dz zztdgz}|d||zj ddz|j dz S)angroupmeans using dummy variables Parameters ---------- x : array_like, ndim data array, tested for 1,2 and 3 dimensions d : ndarray, 1d dummy variable, needs to have the same length as x in axis 0. Returns ------- groupmeans : ndarray, ndim-1 means for each group along axis 0, the levels of the groups are the last axis Notes ----- This will be memory intensive if there are many levels in the categorical variable, i.e. many columns in the dummy variable. In this case it is recommended to use a more efficient version. rN).Nr?)rasarrayndimr&sum)r!rgrislis r groupmean_drsv2 1 A FFQJE ;-4&%'* *eDk] :C hK!C& % %a ( +AEE!H 44rc6eZdZdZd dZdZdZdZdZdZ y) TwoWayaa wrapper class for two way anova type of analysis with OLS currently mainly to bring things together Notes ----- unclear: adding multiple test might assume block design or orthogonality This estimates the full dummy version with OLS. The drop first dummy representation can be recovered through the transform method. TODO: add more methods, tests, pairwise, multiple, marginal effects try out what can be added for userfriendly access. missing: ANOVA table NcP|jd|_|d}d}n|\}}t||x\|_|_\}}t|x\|_|_\} } |jdx|_} | jdx|_} t|| } | \}}}}}}| \|_ |_ |_ |_ |_}t|| d}t|| d}t!|||_|jd|_||_t)j*||j-|_|j.j0|_|j"j3|j0|_d| dz z| dz z|_|j$|j6z |_y)NrrrrrVr^rY)r\rhr7rJ d1_labelsrK d2_labelsnlevel1nlevel2rM prod_labelC1C1_labelC2C2_labelr8r} transformriexogsmOLSfitresolsparamsr params_dropfstart_interaction n_interaction)rendogfactor1factor2varnamesvname1vname2rJrrKrrrresprodlabrC1labrC2labr3dp_fulldp_dropfs rrzTwoWay.__init__sMM!$  FF%NFF2:7F2KK-"i2:7F2KK-"i!#!, w!#!, wy)4+.(UBqMPJ$-$-Bv6 R =':]]1%  ffUG,002 kk(( !NN77 D!"gk!2gk!B!ZZ$*@*@@rc|j}tjtj||j|z ftj |f}|j j|}||_|S)z7contrast/restriction matrix for no interaction ) rrhstackr rirrrR_nointer_transf)rnia R_nointerrs r r_nointerzTwoWay.r_nointersc  IIrxxdjjn(=>s LM >>77 B 0rc0|j}tjtj||j|z ftj |f}|j j|}||_|jj|}|Sz1ttests for no-interaction terms are zero ) rrrr rirrrrrt_test)rrrrt_ress rttest_interactionzTwoWay.ttest_interactionsw  IIrxxdjjn(=>s LM >>77 B 0 ""#34 rcX|j}|jj|Sr)rrf_test)rrs rftest_interactionzTwoWay.ftest_interactions' >>+{{!!"233rc|dk(r1|jj|j|jfS|jj|j|j fS)Nr)rrrrrr)r factorinds rttest_conditional_effectzTwoWay.ttest_conditional_effectsM >;;%%dgg. = =;;%%dgg. = =rcddlm}|jj|j|j }|j }|j}d}tdg|j z}||||||S)Nr) SimpleTablez!Estimated Coefficients by factorsz%#10.4g) data_fmts)titletxt_fmt) statsmodels.iolibrrr[rrrrdict)rr params_arrstubsheadersr table_fmts r summary_coeffzTwoWay.summary_coeffsj1[[((t||D ..3" DLL02 :wU#,. .rN) rrrrrrrrrrrrrrrs'&AD  4 > .rrc$eZdZdZdZdZdZy)TestContrastToolscgd|_ddg|_tjgdgdgdgdgdgdgdgdgdgdgdgdg |_y)Na0a1a2b0b1)rrr)rrr)rrr)v1namev2namerr rJ)rs rrzTestContrastTools.__init__.sL( Tl ((I%%%%%%%%%%% ' (rctjgdd}tjddgddgddgddgddgddgddgddgddgddgddgddgg ddg}}t|d \}}t||t||y) N) FrMrrrrrrrrrz|S1)dtyperrgender_Fgender_Mgender)rT)rr r7r)rr!rgr0res_d res_labelss r test_dummy_1dzTestContrastTools.test_dummy_1d>s HHQ XX1v !1v !1v !1v !1v !1v !1v !1v !1v !1v !1v !1v ' (+5j)A %Q9zUAZ(rct|j|j}dgdz}gd|d<tjgdgdgdgdg|d<gd |d <tjgd gd gd g|d<gd|d<t dD]3}tj j||||t|5y)Nr)a0_b0a0_b1a1_b0a1_b1a2_b0a2_b1)rrrr)rrrrrr)rrrrrr)rrrrrrr)z a1_b0-a0_b0z a1_b1-a0_b1z a2_b0-a0_b0z a2_b1-a0_b1r)rrrrrr)rrrrrr)rrrrrr)z a0_b1-a0_b0z a1_b1-a1_b0z a2_b1-a2_b0)err_msg) rMrrrr rtestingrr )rres_cpres_tiis rtest_contrast_productz'TestContrastTools.test_contrast_productQs!$++t{{;AIa88;999;<aPaHH<99;<aAa( LB JJ # #F2Jb 3r7 # K Lrct|j\}}t|tjgdt|tjgdy)N)rr)rr  )rqrJrrr )rres rtest_dummy_limitsz#TestContrastTools.test_dummy_limitsas:477#!Q+,Q./rN)rrrrrr rrrrrr,s( )&L 0rr__main__)smalllargeNrrrrr rrrrrrbrrXrYrg{Gz?gMbP?g?rrz" tvalues for no effect of factor 1z/each test is conditional on a level of factor 2z" tvalues for no effect of factor 2z/each test is conditional on a level of factor 1)rr)F)NNFr)rV)hr numpy.testingrnumpyrrrrr#r1rMr7r8rqr{r}rrrrttr rrstatsmodels.apiapirexamplesrrrprintrPyx1x2rJrrKrrepeatr\rhdd_fulldd_dropldd_dropfrTdiagrrf effect_size noise_scalerandomrandnnoisebeta ydata_full ydata_dropl ydata_dropfrrresols_full_fullresols_full_dropfr params_f_f params_f_dfresols_dropf_fullresols_dropf_dropf params_df_f params_df_dfrrtr_oftr_fo transf_f_dfrOabsrrrrrrrrr3rtvalue cov_paramsrr rR_nointtvalues inter_direct inter_transftwrpvaluerfvaluerrrrrrCsB0', ,<ED4