%gG@dZddlZddlZddlZddlmZmZmZmZm Z m Z m Z ddl Z ddlZddlmZmZmZ d$dedede ede ed ed e ed eeeffd Zd%deded eeeffd Zd%deded eeeffdZded eeeffdZded eeeffdZde deeefded efdZde deeefded efdZde deeefded efdZded efdZdeed eefdZ de deeefded efdZ!de deeefdeded ef dZ"ded eefdZ#ded eefdZ$d ed e eee ee%e jLfffd!Z'd"ed e ee%e jLffd#Z(y)&z3 Utility functions for reading the Stan CSV format N)AnyDictListMutableMappingOptionalTextIOUnion)_CMDSTAN_SAMPLING _CMDSTAN_THIN_CMDSTAN_WARMUPpathis_fixed_param iter_sampling iter_warmup save_warmupthinreturncdt||}|t}nO|tkDrFd|vrtdj|||d|k7rtdj|||d|}|t}|}|t }t tj||z }t tj||z }|d|k7rtdj|||d|rLd|vr|ddvstdj||d |k7rtd j|||d |S) z3Capture essential config, shape from stan_csv file.rz6bad Stan CSV file {}, config error, expected thin = {}z@bad Stan CSV file {}, config error, expected thin = {}, found {}draws_samplingz1bad Stan CSV file {}, expected {} draws, found {}r)truez  #N*L& tyy!456L>D#89:N / ? F Fnd+;&<   %$}*=*L99?   < /55;V,^(<6  Kci}d}t|d5} t|||}t|||}|st|||}t |||}t ||||} ddd|S#t $r}t d|z|d}~wwxYw#1swY|SxYw)z2Process sampler stan_csv output file line by line.rrzError in reading csv file: N)open scan_configscan_column_namesscan_warmup_itersscan_hmc_paramsscan_sampling_itersr)r rdictlinenofdes r rrFsD F dC JB J T62F&r48F!*2tV<(T6:(T6>JF J K J:TAB I J J Ks)BAA"" A?+A::A??BB  save_itersc i}d}t|d5}t|||}t|||}d}|D]}|dz }  ddd|r*tjt |dft d}t|d5}t|D]}|jtD]}|jj}t |dkrtdj|||z|jd } |r | D cgc] } t | c} |ddf<||dz k(stj| t }  ddd |d <|r|d <|S#1swY+xYwcc} w#1swY)xYw) z4Process optimizer stan_csv output file line by line.rr$rN column_namesF)dtypeorderz*cannot parse CSV file {}, error at line {},)r3mle all_iters)r%r&r'npemptylenfloatrangereadlinestriprrsplitarray) r r/r+r,r-itersliner7ixsxr6s r scan_optimize_csvrFWsD F dCBRv."2tV4 D QJE    " C^,- .e3!  dCr:rWendswithreplacerUr?rrr;seek)r-r]r,cur_posrBkey_valraw_valvals r r&r&s ggiG ;;=   D d)a-DOOC0!  == %<< R0D{{6"**S/ w<1 qz!V+GAJ4G4G4N+21:+;+;+= K(!!#v-!!***, *g,C36 GAJ,,./''){{}""$5 d)a-DOOC06GGG M" **#Gn%*"f,"#C$/"#C")C * *s6 F G F,+G,G G G  GGcd|vr|S|j}|jj}d}t|dkDri|j dsX|dz }|dz }|j}|jj}t|dkDr|j dsX|j |||d<|S)z* Check warmup iterations, if any. rrrNrr)rer=r>r:rWrh)r-r]r,rirB draws_founds r r(r(s K' ggiG ;;=   DK d)a- 4! q ''){{}""$ d)a- 4 GGG"-K Mr"c|jj}|dz }|j|d<|jd}tt ||d<|S)zF Process columns header, add to config_dict as 'column_names' r raw_headerr5r1)r=r>r?tuplemunge_varnames)r-r]r,rBnamess r r'r'sW ;;=   D aKF $ K  JJsOE"'u(=">K Mr"namecd|vrd|vr|S|jd}t|D]9\}}d|vr |jddd}|jdd}|dz }|||<;dj|S)N.:[rr5])r? enumeratergjoin)rt tuple_partsrCparts r munge_varnamer~s $3d? **S/K[)4 d? ||Ca(||C%   A  88K  r"rscX| td|Dcgc] }t|c}Scc}w)z Change formatting for indices of container var elements from use of dot separator to array-like notation, e.g., rewrite label ``y_forecast.2.4`` to ``y_forecast[2,4]``. zmissing argument "names")rr~)rsrts r rrrrs.  }344,1 2DM$  22 2s'cj|d}|jj}|dz }|dk(stdj|||jj}|dz }|j d\}}|j dstdj|| t |j|j}|jj}|dz }|d k(r)|j d r|S|j||dz S|d k(r|d k(s$|dk(r|dk(stdj||jjd}|dz }t|j d} |d k(r|Std| D]]} |jjd}|dz }t|j d| k7sEtdj||S#t$r!}tdj|||d }~wwxYw)zC Scan step size, metric from stan_csv file comment lines. metricrz# Adaptation terminatedz(line {}: expecting metric, found: "{}"rLz # Step sizez+line {}: expecting step size, found: "{}"zline {}: invalid step size: {}Nunit_ez# No free parametersdiag_ez+# Diagonal elements of inverse mass matrix:dense_ez"# Elements of inverse mass matrix:z5line {}: invalid or missing mass matrix specificationrJr5) r=r>rrr?rWr;rerhrUr:r<) r-r]r,rrBlabel step_sizer. before_metricnum_unconstrained_paramsr[s r r)r)s@  "F ;;=   D aKF , , 8 ? ? M   ;;=   D aKFzz#E9   M * $fVT2   ioo GGIM ;;=   D aKF  ??1 2M GGM "A:  h EE i D,P$P "F6N   ;;=   'D aKF"4::c?3  q23 A;;=''/D aKF4::c?#'?? $$*F6N   S  , 3 3FI F  s-H H2H--H2c d}t|d}|s1|djd}|djd}|d}d} d} |j} |jj } t| dkDr| j ds|dz }|dz }| j d} t| |k7r7td j||t| j dd z|j} |jj } |s' t| z } t| k(r dz } t| dkDr| j ds|j| ||d <|s  |d < |d <|S)z Parse sampling iteration, save number of iterations to config_dict. Also save number of divergences, max_treedepth hits rr1 divergent__ treedepth__ max_depthrNrr5z0line {}: bad draw, expecting {} items, found {} zThis error could be caused by running out of disk space. Try clearing up TEMP or setting output_dir to a path on another drive.rct_divergencesct_max_treedepth) r:indexrer=r>rWr?rrrrh)r-r]r,rrnnum_cols idx_divergent idx_treedepth max_treedepthrrrirBdatas r r*r*RsK;~./H #N399-H #N399-H #K0 ggiG ;;=   D d)a- 4! q zz# t9 CJJHc$**S/&:%% ''){{}""$ c$}"56 6N4 &'=8 A% % d)a- 4(GGG$/K ! (6 $%*: &' Mr"c|jdrut|d5}tj|}ddddvr-t j |d}t |jStdj|t t|}|tdj||S#1swYxYw)z` Read metric file in JSON or Rdump format. Return dimensions of entry "inv_metric". z.jsonr$N inv_metric1metric file {}, bad or missing entry "inv_metric") rfr%jsonloadr8asarraylistshaperrread_rdump_metric)r r- metric_dictdims_npdimss r read_metricrs  }}W $_ ())B-K ( ; &"$**[-F"GG & &&&,fTl  %d+, <&&,fTl  # ( (s B99Cct|}|!d|vrt|dtjst dj |t |djS)zL Find dimensions of variable named 'inv_metric' in Rdump data file. rr)rload isinstancer8ndarrayrrrr)r rs r rrsa+K # {<0"** = ? F Ft L    L)// 00r"fnameci}t|d5}|j}dddd}|tkr"d||vr|dz }|t|krd||vr|t|k(ry|}|dz } |t|kr"d||vr|dz }|t|krd||vr|}dj|||j dd}|j dDcgc]}|j c}\} } | j dd} | j d d} t| || <|t|k(r |S|}|dz }#1swY$xYwcc}w) aParse data and parameter variable values from an R dump format file. This parser only supports the subset of R dump data as described in the "Dump Data Format" section of the CmdStan manual, i.e., scalar, vector, matrix, and array data types. r$Nrz<-rr_ "L)r% readlinesr:r{rgr?r>parse_rdump_value) r data_dictr-linesidx start_idxnext_varvar_dataitemlhsrhss r rrsw I eS R  C E t5:5 q E t5:5 c%jI1HC CJ4uSz#9 1HCCJ4uSz#9775845==dBG-5^^D-ABTDJJLBSkk#r"kk#r"*3/ # #e*     q CsEEE rctjd} |jdr|j|}||j d t ||j dj dDcgc] }t|}}tj|d}|j d|j dj dDcgc] }t|}}tj|j|d} |S|jd rQ|jd r@tj|d d j dDcgc] }t|c}}|Sd |vsd|vr t|}|St|}|Scc}wcc}wcc}w#t$r }t dj||d}~wwxYw)zProcess right hand side of Rdump variable assignment statement. Value is either scalar, vector, or multi-dim structure. Use regex to capture structure values, dimensions. zQstructure\(\s*c\((?P[^)]*)\)(,\s*\.Dim\s*=\s*c\s*\((?P[^)]*)\s*\))?\) structureNvalsr5r2)r4rzc()r`rvr.zbad value in Rdump file: {})recompilerWmatchgrouprr?r;r8r@rreshaperf TypeErrorr) rpatparsevrrlrrr.s r rrs ** ; C K >>+ &IIcNE} F 3 ; o%&+kk&&9&?&?&DEE!HEDE((4s+C{{6".(- F(;(A(A#(FG1AGGhhtn,,T,= J^^D !cll3&7((C"IOOC4HIDE$KIJC J CZ3#:*C Jc(C JFHJ K6==cBCJKsVA"F'9F A F'F)(F'AF'F"-F'6F' F'F'' G0G  G)FNNFN)F))__doc__rrrtypingrrrrrrr numpyr8pandasrX cmdstanpyr r r strboolrr!rrFrHr\r&r(r'r~rrr)r*rrr;rrrr"r rs KKKGG !#'!% 4 44C=4# 4  4 3- 4 #s(^4n3c3h"$C$T$d38n$N34S> s tCH~ F#F#c3h###L!#s(^58*  +CH5 ?B   ! ! ! 3$s)3S 3@@!#s(^@58@@F++!#s(^+58+JN++\cd3i4 1C 1DI 1$sE#ubjj2H,I'I"JKB35eRZZ)?#@r"