
    Ej}                        d dl mZ d dlmZ d dlZd dlZd dlZd dl	Z	d dl
Zd dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZmZmZ d dlmZmZmZmZmZ  G d d	          Z dS )
    )requests)iscloseN)sharedutils)YfConfig)
_BASE_URL__PRICE_COLNAMES_period_default)YFDataExceptionYFInvalidPeriodErrorYFPricesMissingErrorYFRateLimitErrorYFTzMissingErrorc                   0   e Zd ZddZej        eddddddddddddfdej        fd            Z	ddej        fd
Z
defdZddej        fdZddej        fdZddej        fdZd dej        fdZddej        fdZej        d!d            Zd Zd"dZej        d             Zej        d             Zej        d             Zej        d             Zej        d             Zej        d             Zej        d             Zej        d#d            ZdS )$PriceHistoryNc                     || _         |                                | _        || _        |pt	          j        d          | _        i | _        d | _        d| _	        d | _
        d S )Nchrome)impersonateF)_datauppertickertzr   Sessionsession_history_cache_history_metadata_history_metadata_formatted_reconstruct_start_interval)selfdatar   r   r   s        U/var/www/html/bolsaweb/venv/lib/python3.11/site-packages/yfinance/scrapers/history.py__init__zPriceHistory.__init__   sa    
llnnH("2x"H"H"H !%+0( ,0(((    1dFT
   returnc           
      H0  < t          j                    }|rt          j        dt          d           |}|t
          k    rd}|s|rd}nd}n|}|	r|dv r|dk    rt          d          |||| j        }|t          | j	                  }t          |          }t          j                    t          j        | j	        <   |                    d	d
          d
         t          j        | j	        <   |st           j        j        s||                    |           t          j                    S |dk    rLt)          j        t,          j                            d                              |          j        d
d
          }nst,          j                            d                              |                                          }|t          j        |          z  }|t)          j        d          z  }|}d}d}|}|}|s|s|r|                                dk    r| j        }|t          | j	                  }t          |          }t          j                    t          j        | j	        <   |                    d	d
          d
         t          j        | j	        <   |st           j        j        s||                    |           t          j                    S |r6t          j        ||          }t?          |                                           }|r6t          j        ||          }t?          |                                           }||s|sd}nv|s:|t          j        d          z
  }t?          |                                           }n:|sSt,          j                            d                              |          }t?          |                                           }n|                                dk    rO| t?          tC          j"                              }|*|dk    r|dz
  }n|dv r|dz
  }n|dv r|dz
  }n|dz
  }|dz  }n}|r|rt          d          |s|rft          j        |          }|&||z   }t?          |                                           }|&||z
  }t?          |                                           }d}|s|r||d}n|                                }d|i}|                                |d<   ||d<   |d         dk    rd|d<   d|d <   tG          |          }| j        }dD ]X}||v rRt          t-          j        ||         d!"          $                    d                              |                    ||<   Y|                    | j	         d#t          |                      tJ           d$| j	         }d}| j&        j'        }|rt-          j        |d!"          $                    d          }t,          j                            d          }t)          j        d%&          } || z   |k    r| j&        j(        }	  ||||'          }d(|j)        v s|tU          d)          |+                                }n.# tX          $ r  tZ          $ r |st           j        j        s Y nw xY w|pi '                    d*          pi }!|!'                    d+          }"t]          |"t^                    r5ta          |"          d,k    r"|"d,         pi }#|#'                    d-          pi }$ni }$|$| _1        |d         d.         d/v }%d0}&|s||                                dk    rW|&d1|d          d2z  }&||&| z  }&n|%sS|&t-          j        |d!"          $                    d                              |                                           z  }&n@|&t-          j        |d!"          $                    d                              |           z  }&|&d3z  }&|	|&| d4z  }&n|%sU|&t-          j        |d!"          $                    d                              |                                           d4z  }&nL|&t-          j        |d!"          $                    d                              |           d4z  }&n	|&d5| d4z  }&d6}'|t]          |tF                    ste          | j	        |&          }d7}'nAt]          |tF                    r,d8|v r(|&d9|d8          d4z  }&te          | j	        |&          }d7}'n d*|v rC|d*         d:         r5|&d;|d*         d:         d<         z   d=z   z  }&te          | j	        |&          }d7}'nd*|vsB|d*         d+         4|d*         d+         r&|d*         d+         d,         d>         d?         d,         ste          | j	        |&          }d7}'n[|rY|| j1        d@         vrJt          j3        |          s6ti          | j	        |dA5                    | j1        d@                             }d7}'|'rt          |          }t          j                    t          j        | j	        <   |                    d	d
          d
         t          j        | j	        <   |st           j        j        s||                    |           | j6        | j6        |k    rd| _6        t          j                    S | j1        dB         }(|(dCv })| j1        dD         }*| j1        dE         }+|r|| j1        '                    d@g           vrt?          tC          j"                              }t-          j        |d!"          $                    d          }t(          j        7                    |          }|t          j        |          z  }|t)          j        d          z  }t          j8        |d*         d+         d,                   },|r^|,j9        sW|,j:        d.         |                    d          $                    d          k    r |,;                    |,j:        d.                   },|,j9        r| j	         dF}-nQta          |,          d
k    r| j	         dG|,j:        d,          dH}-n%| j	         dG|,j:        d,          d3|,j:        d.          }-|                    |-           |                                dk    r|                    | j	         dI           |,<                    dJ          }.t-          j=        |.>                                j:        |.dK         ?                                |.dL         @                                |.dM         A                                |.dN         >                                |.dO         >                                |.dP         B                                dQR          },t          jC        |,||*          },t          jD        |,|          },|d         d.         d/v }%|s||%rzdS| j1        v rq| j1        dS         }/t]          |/t,          j=                  s4t          jE        | j1        d7T          | _1        d7| _F        | j1        dS         }/t          jG        |,||/          },|,j9        r| j	         dU}-nQta          |,          d
k    r| j	         dV|,j:        d,          dH}-n%| j	         dV|,j:        d,          d3|,j:        d.          }-|                    |-           t          jH        |d*         d+         d,                   \  }0}1}2|)sd}2|17t          jC        |1||*          }1|1dW         I                    dX          | _J        nt-          jK                    | _J        |0t          jC        |0||*          }0dE|0jL        v r|0I                    dX          | _M        | j1        dE         }3|3d0}3|0dE         |3k    }4|4N                                r|	r|3d0k    r| O                    |0|3|	          }0|0;                    dEd
Y          }0n9|0dZ         I                    dX          | _M        nt-          jK                    | _M        |27t          jC        |2||*          }2|2d[         I                    dX          | _P        nt-          jK                    | _P        |Z|,j9        sS|,j:        d,         Q                    d\          }5|0|0jR        |5d         }0|2|2jR        |5d         }2|1|1jR        |5d         }1|;|t-          jS        d
          z
  }6|0
|0d|6         }0|2
|2d|6         }2|1
|1d|6         }1|d         d.         d/v }%|%st-          jT        |,j:        j                  $                    |*d7d]^          |,_:        |0jU        d,         d,k    r9t-          jT        |0j:        j                  $                    |*d7d]^          |0_:        |1jU        d,         d,k    r9t-          jT        |1j:        j                  $                    |*d7d]^          |1_:        |,V                                <|0jU        d,         d,k    rt          jW        <|0|          <dZ<jL        v r%d,<jR        <dZ         X                                dZf<   nd_<dZ<   |1jU        d,         d,k    rt          jW        <|1|          <dW<jL        v r%d,<jR        <dW         X                                dWf<   nd_<dW<   |)rZ|2jU        d,         d,k    rt          jW        <|2|          <d[<jL        v r%d,<jR        <d[         X                                d[f<   nd_<d[<   <j9        r| j	         d`}-nQta          <          d
k    r| j	         da<j:        d,          dH}-n%| j	         da<j:        d,          d3<j:        d.          }-|                    |-           t          jY        <|d         |*||	|+b          \  <}7|7|7dN         |7jZ        dc| j1        dd<   <<j:        [                    def                    <|	rI|                    | j	         dg           <V                                <| \                    <|+          \  <}+|+| j1        dE<   | ]                    <||+          <<j9        sg| ^                    <j_        d.d         ||*|          }8dh<jL        vrd6<dh<   t-          j`        <;                    <j:        d.                   |8g          <| a                    <||*|          <| b                    <||*          <| ^                    <||*|          <| c                    <          <<V                                <	 |rt          jd        <          <n|rt          je        <          <n# tZ          $ r}9|st           j        j        s |rdi|9z  }ndj|9z  }t          j                    t          j        | j	        <   |t          j        | j	        <   |                    | j	        d	|           Y d}9~9nd}9~9ww xY w|r3t          jg        <|d*         d+         d,         d-         dk                   <<dP         h                    d,          i                    t          jj                  <dP<   |%rdl<j:        _Z        ndX<j:        _Z        |s<;                    g dmdno          <|
syt          dPgz   g dmz   }:<fdp|:D             }:<|:         X                                <|:         d,k    z  l                    d
Y          };<;                    |;j:        |;                   <||k    r| m                    <|||          <<j9        r| j	         dq}-nQta          <          d
k    r| j	         dr<j:        d,          dH}-n%| j	         dr<j:        d,          d3<j:        d.          }-|                    |-           <n                                <| j6        | j6        |k    rd| _6        <S )saJ  
        :Parameters:
            period : str
              | Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
              | Default: '1mo' if start & end None
              | Can combine with start/end e.g. end = start + period
            interval : str
              | Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
              | Intraday data cannot extend last 60 days
            start : str
              | Download start date string (YYYY-MM-DD) or _datetime, inclusive.
              | Default: 99 years ago
              | E.g. for start="2020-01-01", first data point = "2020-01-01"
            end : str
              | Download end date string (YYYY-MM-DD) or _datetime, exclusive.
              | Default: now
              | E.g. for end="2023-01-01", last data point = "2022-12-31"
            prepost : bool
              | Include Pre and Post market data in results?
              | Default: False
            auto_adjust : bool
              | Adjust all OHLC automatically?
              | Default: True
            back_adjust : bool
              | Back-adjusted data to mimic true historical prices
            repair : bool
              | Fixes price errors in Yahoo data: 100x, missing, bad dividend adjust.
              | Default: False
              | Full details at: :doc:`../advanced/price_repair`.
            keepna : bool
              | Keep NaN rows returned by Yahoo?
              | Default: False
            rounding : bool
              | Optional: Round values to 2 decimal places?
              | Default: False = use precision suggested by Yahoo!
            timeout : None or float
              | Optional: timeout fetches after N seconds
              | Default: 10 seconds
            raise_errors : bool
                If True, then raise errors as Exceptions instead of logging.
        zF'raise_errors' deprecated, do: yf.config.debug.hide_exceptions = False   )
stacklevelN1mo)5d1wkr*   3mor+   z<Yahoo's interval '5d' is nonsense, not supported with repair:    ytdUTC   daysr$   max1mi 
 )2m5m15m30m90mi O )1h60mi gl   j-t zASetting period, start and end is nonsense. Set maximum 2 of them.)period1period2rangeintervalincludePrePostr:   r9   zdiv,splits,capitalGainseventss)unitz: Yahoo GET parameters: z/v8/finance/chart/   )minutes)urlparamstimeoutzWill be right backz)*** YAHOO! FINANCE IS CURRENTLY DOWN! ***chartresultr   metamh  (  -> )z	 (period=FTstatus_codez(Yahoo status_code = errorz (Yahoo error = "descriptionz")
indicatorsquotevalidRanges, instrumentType)
MUTUALFUNDETFexchangeTimezoneNamecurrencyz$: yfinance received OHLC data: EMPTYz: yfinance received OHLC data: z onlyz: resampling 30m OHLC from 15m30minOpenHighLowClose	Adj CloseVolume)rd   re   rf   rg   rh   ri   )indexr    tradingPeriods)tradingPeriodsOnlyz: OHLC after cleaning: EMPTYz: OHLC after cleaning: Stock SplitsDateaxis	DividendsCapital GainsDshift_forward)	ambiguousnonexistent        z$: OHLC after combining events: EMPTYz: OHLC after combining events: )repairrb   )PriceTime	lastTradefirst)keepz: checking OHLC for repairs ...	Repaired?zauto_adjust failed with %szback_adjust failed with %s	priceHintDatetimerq   rm   rr   ignore)columnserrorsc                 &    g | ]}|j         v |S  r   .0cdfs     r!   
<listcomp>z(PriceHistory.history.<locals>.<listcomp>  s     III1bjQr#   z : yfinance returning OHLC: EMPTYz: yfinance returning OHLC: )or   get_yf_loggerwarningswarnDeprecationWarningr
   
ValueErrorr   r   r   strempty_dfr   _DFSsplit_ERRORSr   debughide_exceptionsrX   	_datetimedatepd	Timestampnow
tz_convertyear_interval_to_timedelta	timedeltalower_parse_user_dtint	timestamp_timetimedicttz_localizer   r   get	cache_gettextr   jsonr   	Exception
isinstancelistlenr   r   is_valid_period_formatr   joinr   fromtimestampparse_quotesemptyrj   dropresample	DataFramelastr|   r5   minsum	set_df_tzfix_Yahoo_dst_issueformat_history_metadatar   'fix_Yahoo_returning_prepost_unrequestedparse_actionsrename_axis_splitsSeriesr   
_dividendsany_dividends_convert_fx_capital_gainsfloorloc	Timedeltato_datetimeshape
sort_indexsafe_merge_dfsisna!fix_Yahoo_returning_live_separatename
duplicated_standardise_currency_fix_bad_div_adjust_fix_zeroesilocconcat_fix_unit_mixups_fix_bad_stock_splits_repair_capital_gainsauto_adjustback_adjustnproundfillnaastypeint64r	   all	_resample_consolidate)=r   periodrA   startendprepostactionsr   r   rx   keepnaroundingrJ   raise_errorsloggerinterval_userperiod_userr   
_exceptionerr_msg
start_userend_userstart_dtend_dt	period_tdrI   params_prettykrH   r    get_fndt_now
data_delay
safe_chartresult_list
first_itemrM   intraday_price_data_debugfail
quote_typeexpect_capital_gainstz_exchangerb   quotesmsgquotes2tps	dividendssplitscapital_gainsprice_currencyf_currency_mismatchstart_dend_dt_sub1
last_tradedf_lastedata_colnamesmask_nan_or_zeror   s=                                                               @r!   historyzPriceHistory.history   s   ^ $&& 	FMbdv  DE  F  F  F  F ^##K   K 	h"=== 4 !_```}1CW:!1$+!>!>J!*ooG/4~/?/?FK,29--a2H2H2KFN4;/# .HN,J .((W--- >+++U??%N2<+;+;E+B+B+M+Mb+Q+Q+VXY[\]]EEL,,U33>>rBBGGIIEU9&AAAEY0a8888E$H
 	(C 	(F 	(v||~~'>'>Bz-dk::
j//+0>+;+;DK(.5mmD!.D.DQ.Gt{+ *(F *$$LL)))~''' 	.+E266H**,,--E 	*)#r22Ff&&(())C> .S . .!E$@$G$GGH..0011 .))%00;;B??&**,,--||~~&&;ejll++C=4'' #f!%FFF #g!]22 #h #j 0QJE 
3 
 !deee # !8@@	;%	1Ff..0011C=%	1H 2 2 4 455E 	'C 	'!&377FF\\^^Fv&F%^^--z#*  *&&!&F: 5xVW' 	l 	lAM!!#&r|F1IC'H'H'H'T'TUZ'['['f'fgi'j'j#k#ka QQS=O=OQQRRR <<t{<<?\#C000<<UCCF\%%e,,F",R888J
"f,,-	6  D
 $ty00DL%&QRRR99;;DD 	 	 	 	 	 	 HN$B  	
 zr&&w//52
 nnX..k4(( 	S-=-=-A-A$Q-2J>>&))/RDDD!%*%b)Z7 	7FNfllnn&=&=!;fZ&8!;!;!;;%!
_4!! j!U(E(E(E(Q(QRW(X(X(c(cdf(g(g(l(l(n(n%pp!!!U(E(E(E(Q(QRW(X(X(c(cdf(g(g%ii!'#!^^^3!! i!Ss(C(C(C(O(OPU(V(V(a(abd(e(e(j(j(l(l%o%o%oo!!!Ss(C(C(C(O(OPU(V(V(a(abd(e(e%h%h%hh!!!6V!6!6!66<z$55<-dk;LMMJDDd## 	(=(=!Om9L!O!O!OO-dk;LMMJDD__gw!7_!4tG}W7Mm7\!\_c!cc-dk;LMMJDDD  DM($;$C4PW=YaKb$Cjnovjw  yA  kB  CD  kE  FR  kS  T[  k\  ]^  k_$C-dk;LMMJDD 	d&<]&KKKTYTpqwTxTxK-dk6499TMcdqMrCsCsttJD 
	$*ooG',~'7'7FK$*1--a*@*@*CFN4;' &HN$B &  W%%%/;@`dl@l@l370>### +,<=
)-BB,-CD)*5  	1fD$:$>$>}b$Q$QQQejll##C\#C000<<UCCFN0055EU1&999EY(a0000E #DM($;A$>?? 	7v| 	7|B6#4#4U#;#;#G#G#M#MMMV\"%566< 	i[FFFCC[[A[WWaWWWCC[hhahhV\VbceVfhhCS >>u$$LLDKGGGHHHoog..G\(<--//++--u~))++ )..00$[16688!(+//11D D   F ;??*68<<*%b)Z7 	Z8 	Z(8D<R(R(R()9:Cc2<00 ?).)FtG]rv)w)w)w&370,-=>B68UXYYF< 	a[>>>CC[[A[OOaOOOCC[``a``fl[]N^``CS ,1+>tG}X?VWX?Y+Z+Z(	6=# 	! M_VX{CCF!.1==fEEDLL9;;DL 	8[IIIY... #,"7"7"?"?!%!7
!C!)%'N&/
&;~&M#&**,, b b.B"6"6$($>$>y.Z`$a$a	%NN:AN>>		"+K"8"D"DV"L"L ikkDO$!OM8[QQM"/"@"L"LV"T"TD"$)++D< 2 ,q///44( )ghh 7I ,$1$5ghh$?M%#Z1F? 2<??2K$%l{l3	( -l{l ;!- *%b)Z7 	G >&,*;<<HH`d  sBH  C  CFLq!A%%"$.1E"F"F"R"RS^jn  }L"R  #M  #M	|A""!~fl.?@@LL[dh  wFL   G   G   ?1!!%b)X>>B"*$$:;BF2k?''));677!B{O<?Q%b&(;;BRZ''@ABF2n%**,,n<==!$B~ 	*"1%)))"mXFF"*,,FGr/*//11?BCC&)?#8 	a[FFFCCWW\\[SS!SSSCC[``!``RTRZ[]R^``CS@VJEWYdfmv|  HP  Q  Q  QJ!;Eg;NWaWf2g2gD";/$$'$2223 	!LLDKHHHIIIB
  55b(CCLB19D":.))"hAAB 8 A**27233<;PWXXbj00&+B{OY 5 5w?@@&&r8['JJB++B+FFB!!"hWEEB ++B//BB	< +&r** +&r** 		< 		< 		< HN$B  ;6:6:',~'7'7FK$*1FN4;'LLT[[['':;;;;;;;;		<  	O"d7mH5a8@MNNB(|**1--44RX>>8 	#&BHMM"BHM  	b!O!O!OX`aaB 	C,z9<j<j<jjMIIIIIIIM "= 1 6 6 8 8B}<MQR<R SXX^_X``)/0@ABBB}$$Hm[IIB8 	][BBBCCWW\\[OORXa[OOOCC[\\RXa[\\bhWYl\\CS __+7D<\`h<h<h/3D,	s3   <Z (Z65Z6V -AV. V.
AX;V8A9AX6X6AX;r5   c                     |||f}|| j                                         vr7|                     |||d          }|| j        | j        | j        d| j         |<   | j         |         S )NT)r   rA   rx   r   )pricesr
  r  capital gains)r   keysr  r   r   r   )r   r   rA   rx   	cache_keyr   s         r!   _get_history_cachezPriceHistory._get_history_cache  sz    vv.	D/446666VhvW[\\B8::>,AEAT.V .VD	* "9--r#   c                     | j         	d| j         vr|                     dd          d          | j        du r%t          j        | j                   | _         d| _        | j         S )Nrk   r+   r<   )r   rA   r  FT)r   r  r   r   r   )r   s    r!   get_history_metadataz!PriceHistory.get_history_metadata$  sl    !)-=TE[-[-[##4$#??II+u44%*%B4CY%Z%ZD"/3D,%%r#   c                 >    |                      d||          d         S )Nr$   rA   r   rx   r
  r  r   r   rx   s      r!   get_dividendszPriceHistory.get_dividends/  s#    &&VF&SST_``r#   c                 >    |                      d||          d         S )Nr$   r   r  r!  r"  s      r!   get_capital_gainszPriceHistory.get_capital_gains2  s#    &&VF&SSTcddr#   c                 >    |                      d||          d         S )Nr$   r   r  r!  r"  s      r!   
get_splitszPriceHistory.get_splits5  s#    &&VF&SST\]]r#   c                    |                      |          }|d         |d         }|t          |t          j                  rd|j        v rt          j                            dd          |d          d                             d	          d<   d                             d
          d<   	                    ddi          g d}fd|D                      g d}fd|D             }|         dk    
                    d                   |D ]7}|         d
k                                    r                    |d          8S )N)r   r  r
  rb   rq   r/   ro   r$   rR   rw   Dividends FXr   )rq   r)  rm   rr   c                 &    g | ]}|j         v |S r   r   r   s     r!   r   z,PriceHistory.get_actions.<locals>.<listcomp>F  s     999Abjar#   r   c                 &    g | ]}|j         v |S r   r   )r   r   r   s     r!   r   z,PriceHistory.get_actions.<locals>.<listcomp>I  s%    HHHa13G3G3G3G3Gr#   r   )r  r   r   r   r   r   r   r   r   renamer   r   )	r   r   r    divscolscols_numericr   r   r   s	          @@r!   get_actionszPriceHistory.get_actions8  s~   &&f&55(^K 
4 > >:QUQ]C]C]%bggkg&B&BD$OOB
^22266BzN o44S99B{OJ#?@@BMMM9999999:EEEHHHH<HHH7<0!388a8@@A 	2 	2A
c!&&(( 2!,,qq,11r#   c                 .   ||k    r|S d }d}|dk    rc|dk    rZd}t          j        t          j                                        j         d          }|                    |j        j                  }nd}n|dk    r`d}|dk    rWt          j        t          j                                        j         d          }|                    |j        j                  }nt|d	k    rd
}nk|dk    rR|dk    rd}	nCt          j                                        	                    d          
                                }	d|	 }nt          d| d          dddddddd}
d|j        v rd|
d<   d|j        v r|
d         |
d<   d|j        v rd|
d<   d|j        |d         dk    df<   |dk    r-|                    |d d |!                              |
          }n,|                    |d d |"                              |
          }d|j        |d         dk    df<   |S )#Nepochr,   r0   7Dz-01-01zW-MONr+   5Dr*   MSr-   JANz%bzQS-z(Not implemented resampling to interval ''r|   r   r5   r   r   prod)rd   rf   re   rg   ri   rq   rm   r~   r   rh   rg   rr         ?rm   rw   left)labelclosedorigin)r;  r<  offset)r   r   r   datetimer   r   r   rj   r   strftimer   r   r   r   r   agg)r   r   df_intervaltarget_intervalr   r>  r=  resample_period
year_startalign_monthresample_mapdf2s               r!   r   zPriceHistory._resampleP  sb   /))Ie##"&\Y-?-C-C-E-E-J*R*R*RSS
#//<<")$$"O\Y-?-C-C-E-E-J*R*R*RSS
#//<<%%"OO%%#'04466??EEKKMM1K11OOZZZZ[[[ E56%  "*$$(-L%"*$$(4W(=L%bj((,1L):=r.!3&67W++oVFSY+ZZ^^_kllCC++oVFSY+ZZ^^_kllC<?N#S(.89
r#   rN   c                 +  O t          j                    }d| j        d}t          t          j                  st          dt                              dk    rS dd          dv rd}d}nd}fd	t          D             }|d
gz   }	g dOfdOD             }
Ofdt          t          O          dz
            D             }d d t          j        d          d}dD ]}t          j        d          ||<   t          j        d          |d<   |v r|         }|
         }n)|                    d d           dj        vrdd<   S | j        | _        | j        k    rQ|| j                 k    r@d                    | j        || j                           }|                    ||           S                                 |	                                         |k    }|                    d          }||         }|d }nl|t          j        d          z  }t          j                            d          |z
  }|                    j        j                                      d          }|                    d| d d| |           |wj        |k    }||z  }|                                sSdt9          j        |                    d                     d }|                    ||           dj        vrdd<   S j        |         }t          |          d!k    r'|                    d"|           dj        vrdd<   S                                 }d|j        vrd|d<   |                                                             d           }||                                         |k                         d          z  }|         }|d!         gg}|d#k    r!tB          j"        "                    d$%          }n|d&k    r!tB          j"        "                    d$%          }n|d'k    r!tB          j"        "                    d$%          }nX|d(k    r!tB          j"        "                    d%          }n1|dk    rt          j        d)          }nt          j        d          }|                    d*| |           t          dt          |                    D ]u}||         }|#                                |d+         d!         #                                |z   k     r|d+         $                    |           _|$                    |g           v|                    d,|           |D ])}|                    d-|d!          d.|d+                     *t          t          |                    D ]_}||         }|d!         }|j        %                    |gd/0          d!         }|d!k    rU||j        |dz
           |k    r?|r8|j        |dz
           #                                |#                                k    r|dz  }|d+         } |j        %                    | gd/0          d!         }!|!t          |          dz
  k     r?|r8|j        |!dz            #                                | #                                k    r|!dz  }!|j        ||!dz            }"||xx         |"&                                z  cc<   ||         '                                 ad!}#|D ]}j        (                    |                   }$|                    d1tS          |$          z              |d!         }%|%#                                }&d}'|d(k    r;t          j#        *                                |&z
  t          j        d2          k    rd}'n>|d3v r:t          j#        *                                |&z
  t          j        d4          k    rd}'|'r#d5|r|%n|& d6}|                    ||           
t          j        d          }(d&v r#|&|z
  })|d+         #                                |z   }*n9d'k    r |&})|d+         #                                |z   }*n|d!         })|d+         |z   }*|)|(z  })|*|(z  }*|r+|)#                                })|*#                                |(z   }*|"tW          |#                                |)          })|                    d7| d8| d9|) d:|* |           t          j                    }tY          |d;          r&|j-        }+|.                    t^          j0                   | 1                    |)|*|dd|dd<          },tY          |d;          r|.                    |+           |,|,j2        r#d5|r|%n|& d6}|                    ||           |,j3        |d!         |d+         |
|         z   t          j        d=          z
                                           },|,j2        r&d> d?|r|%n|& d@}|                    ||           -d!|,dA<   d&k    rmg dB}-|-|$j        d!         4                                dCz   dz
  dCz           }.|,j        5                    d           6                    dD|.z             j7        |,dE<   dE}/nd'k    r$t	          j8        |,j        j#                  |,dF<   dF}/n_d|,j3        |,j        (                    |$j                  dAf<   |,dA         9                                |,dG<   |,:                    dAd          },dG}/|,|,|dHgz                                                                 d                    },|,;                    |/          }0|0<                    dIdJdKdLdMdNdOP          =                    dQdRiS          }1|/dTv r*|1j        5                    |,j        j                  |1_        nl|,dG         >                                |,dU<   t9          j$        |,j        d!         g|,j        |,dG         >                                d!k                       }2|2|1_        |                    dVtS          |1          z              t9          j?        |$j        |1j                  }3t          |3          d!k    r"dW dX|& dY}|                    ||           d'k    r+|1|1j        (                    |3                   }4|$|$j        (                    |3                   }5|5dR         |k    }6|6                                r|5dR         |5dZ         z  }7t8          j@        |7|6<   |7                                                                 s|7A                                B                                }7t9          jC        |6          d!         D ].}8|4j        |8         }t          |7          }9|1j3        |dHf         d!k    rk|8|9dz
  k     r|7jD        |8dz            |7jD        |8<   Td[|4dH         jD        |8         |4dZ         jD        |8dz
           z  z
  }:|7jD        |8dz
           |:z  |7jD        |8<   |8d!k    r|7jD        |8dz
           |7jD        |8<   |7jD        |8dz            |7jD        |8<   |4dH         jD        |8dz            d!k    r@|7jD        |8xx         d[|4dH         jD        |8dz            |4dZ         jD        |8         z  z
  z  cc<   0|5dZ         |k    };|7E                    |$j        t8          j@        \          A                                B                                }7|$dZ         |7z  |1dR<   |;                                r|;E                    |1j        d\          }<|7E                    |1j        t8          j@        \          A                                B                                }=|<                                }>|1dZ         |>         |=|>         z  |1j3        |>dRf<   d]dZg}?|1|1j        (                    |3                   |?                                         }4|$|$j        (                    |3                   |?                                         }5|5|k    }@|@t9          jF        |4           z  }@|@                                s|                    d^|& dY|           	t          t          |?                    D ]q}A|@d d |Af          }B|B                                rN|5jG        jH        s|5                                }5|4jG        jH        s|4                                }4d|5|B|Af<   d|4|B|Af<   r|5|@         |4|@         z  }C|0I                                }D|1j        |D_        |D|Dj        (                    |3                                                   J                    t                    }D|DjG        jH        s|D                                }D|Dd d d f         }Dt9          jL        |Dt          |?                    }D|D|@         }Dt9          jM        |Cd[d_`           }Et9          j        |E          t          |?          k    rd[}Fnt9          jN        |C|Da          }Ft          |Fdbz  dz
            dck     rD|$                                }$t          D ]#}G|j3        ||G         |k    |Gfxx         ddz  cc<   $|Fddz  }F|                    de|Fdf|           t          d[|Fz  d          }Ht          |Fd          }F|Fdk    r|Hdk    rnP|Fdk    r!|1|xx         |Fz  cc<   |1d
xx         |Fz  cc<   n)|Hdk    r#|1|xx         d[|Hz  z  cc<   |1d
xx         |Hz  cc<   |$j        |$|d
gz            |k                                                        d                   }Ig }J|ID ] }8|8|1j        vr|J$                    |8           !t          |J          d!k    r'|                    dgtS          |J          z   |           |ID ]}8|8|1j        vr|1j3        |8         }Kd&k    r7|1jD        |1j        Q                    |8          dz
           }L|,j3        |8d          },j3        |8         }M|Mj        |M|k                                             }Ndh|Nv r|Kdh         |j3        |8dhf<   di|Nv r|Kdi         |j3        |8dif<   d]|Nv rpd&k    rX|8|,j        d!         k    rG|LdZ         |j3        |8d]f<   t          |j3        |8d]f         |j3        |8dif                   |j3        |8dif<   n|Kd]         |j3        |8d]f<   dZ|Nv r%|KdZ         |j3        |8dZf<   |KdR         |j3        |8dRf<   ndR|Nv r|KdR         |j3        |8dRf<   d
|Nv r7|Kd
         P                                J                    dj          |j3        |8d
f<   d|j3        |8df<   |#dz  }#|S )kNzprice-reconstructyf_catyf_interval	yf_symbolz#'df' must be a Pandas DataFrame notr6   r/   )dwkmoTFc                     g | ]}|v |	S r   r   r   s     r!   r   z=PriceHistory._reconstruct_intervals_batch.<locals>.<listcomp>  s    ===AQ"WWaWWWr#   ri   )r,   r$   r<   r:   r9   r8   r7   r6   c                 :    i | ]}|t          j                  S r   )r   r   )r   irA   s     r!   
<dictcomp>z=PriceHistory._reconstruct_intervals_batch.<locals>.<dictcomp>  s&    MMMa5/99MMMr#   c                 4    i | ]}|         |d z            S )r/   r   )r   rS  	intervalss     r!   rT  z=PriceHistory._reconstruct_intervals_batch.<locals>.<dictcomp>  s(    SSSA1yQ/SSSr#   i  r3   )r,   r$   r<   )r:   r9   r8   r7   <   rF   z,Have not implemented price reconstruct for 'z' interval. Contact developersr~   z%Hit max depth of 2 ('{}'->'{}'->'{}')extraro   r1   rs   zmin_dt=z
 interval=z sub_interval=z	Too old (z rows tagged)r   z/Nothing needs repairing (dts_to_repair[] empty)r*      yearsr,   r$   r<   r(   zgrp_max_size = rN   zRepair groups:z- rU   nearest)methodz
df_block:
i  )r:   r9   ;   z"Cannot reconstruct block starting z9, too old, Yahoo will reject request for finer-grain dataz	Fetching z	 prepost=rT   ->level)r   r   rA   r   r   r   rx   r   )millisecondszCannot reconstruct z block range z3, Yahoo not returning finer-grain data within rangectr)MONTUEWEDTHUFRISATSUN   zW-
Week Start	Day Start
intervalIDrq   )rd   r|   )rg   r   )rh   r   )rf   r   )re   r5   )rq   r   )ri   r   )rd   rg   AdjCloserf   re   rq   ri   ro  rh   r   )rl  rm  diffzdf_new:
zCan't calibrate z block starting z so aborting repairrg   r9  
fill_valuerd   zCan't calibrate block starting gh㈵>)rtol)weightsg-C6?{Gz?d   z Price calibration ratio (raw) = 6fz:Yahoo didn't return finer-grain data for these intervals: re   rf   r   )Sr   r   r   r   r   r   r   typer	   r@   r   r   r   warningr   r   formatinfor   to_numpyr   r   r   r   rj   r   ceilr   r   r   copyr   r   	_dateutilrelativedeltar   appendget_indexerto_listsortisinr   todayr5   hasattrra  setLevelloggingCRITICALr  r   r   weekdayr   	to_period
start_timer   cumsumr   groupbyrA  r,  rp  intersect1dnanffillbfillwherer   reindexisnanflags	writeablesizer   floattiler   averageabsr   get_locr   )Pr   r   rA   r   tagr   
log_extrasr   
price_cols	data_colsitdsnextsmin_lookbacksrS  sub_intervaltd_ranger  f_repairf_repair_rowsrP   min_dtf_recentdts_to_repairdf_v2f_gooddf_good
dts_groupsgrp_max_sizedtgg0i0glilgood_dtsn_fixeddf_blockr   r  rejecttd_1dfetch_start	fetch_end	log_leveldf_fineweekdaysweek_end_daygrp_coldf_fine_grpdf_new	new_indexcommon_indexdf_new_calibdf_block_calibf_tagdiv_adjustsidxndiv_adjf_close_badf_close_bad_newdiv_adjusts_newdiv_adjusts_new_np
calib_colscalib_filterjfratiosrt  not1ratior   	ratio_rcpbad_dtsno_fine_data_dts
df_new_rowdf_last_week
df_bad_row
bad_fieldsrV  sP    ``                                                                            @r!   _reconstruct_intervals_batchz)PriceHistory._reconstruct_intervals_batch  s    $&& 3H[_[fgg
"bl++ 	NBDHHMMMtIABB<,,,GHHH====!1===
(+	 HGG	MMMM9MMMSSSSs9~~PQ?Q9R9RSSS $D	8KQT8U8U8UVV+ 	< 	<A(2;;;M!'1r:::du ?LH~HHNNr(rrrsss"*,,"';I +3/7D,t777HdNnHo<o<o9@@Aachim  jJ  dK  MU  V  VCKK:K...I]]__i=))++s2 !,, ,'9FF$!,,,,A\%%e,,q0F&&rx{3388==FWvWWWWWW_ijjjx6)H)H4M $$&& M"&1)=)=">">MMMCz222bj00&+B{O	/}""LLJR\L]]]"*,,"';I		em++!&E+j>&&((,,!,4452j>2244;@@a@HHHV* %Q'()
5  $2@@q@IILLU""$2@@q@IILLT!!$2@@q@IILLT!!$2@@q@IILLT!!$.A666LL$.B777L5|55ZHHHq#m,,-- 	( 	(Aq!Bwwyy:b>!,1133lBBB2%%b))))!!2$''''%Z888 	1 	1ALL/ad//"//0000 s:'' 	! 	!A1A1B**B4	*BB1EBAvvNgmBF&;v&E&E& 'F+2=a+@+E+E+G+G27799+T+T!GB2B**B4	*BB1EBCLL1$$$  W]26%:%?%?%A%ARWWYY%N%N!GB}RQY/HqMMMX--///MMMqM     y	 y	A"(--**+HLLX6777tHmmooGFt##)=)=)?)?')IYM`fiMjMjMj(j(j//Y^5I5I5K5Kg5UYbYlrtYuYuYu4u4u  Xx;\88U\  X  X  XCz222'Q///E5  %0bEJJLL83		T!!%bEJJLL83		dbEH,	 5 KI 5)..00%NN,,u4	!!&++--==LL`\``G``k``U^``hrLsss(**Fvw'' 2"L	 0111ll)lhmw{  FM  VZ  cgl  h  hGvw'' +	***'- Xx;\88U\  X  X  XCz222k!A$"\0B(BYEXfgEhEhEh(h"hinnppG}  ZH  Z  ZxCd88]d  Z  Z  ZCz222GEN5  LLL'):)B)B)D)Dq)H1)LPQ(QR(/(A(A$(G(G(Q(QRVYeRe(f(f(q%&T!!')~gm6H'I'I$%IJGM..x~>>EF(/(=(=(?(?%!,,u1,55&wz[M'ABGGIIMMSTMUUUVG!//'22K __&'."$.( % * * +1&*k9R&*S*S  555%|778HII"),"7"<"<">">Iw}Q'7&8'-P\H]HbHbHdHdghHh:ijj	(LL)CKK7888 >(.&,GGL<  A%%___7___Cz222 4%fl&7&7&E&EF!)(.*=*=l*K*K!L&{3s:99;; 'T"0"=w@W"WK
 *,K&&++--1133  T&1&7&7&9&9&?&?&A&A#%8E??1#5 \ \C!-!3C!8B #K 0 0A%z"k/:a??#&Q;;<G<LSSTW<UK$4S$9$9 /2L4M4RSV4WZfgnZoZt(+a[1 51 /1G<G<LSSTW<UX_<_K$4S$9$9#&77<G<LSSTW<UK$4S$9$9 =H<LSSTW<UK$4S$9'3K'@'EcAg'NRS'S'S(3(8(=(=(=|T_G`GefilmfmGn<H<Q<VWZ<[H\ B\ )\(=(=(=&4W&=&D&1&9&9(.UWU[&9&\&\&b&b&d&d&j&j&l&l.6w.?+.M{+&??,, T.9.A.A&,[`.A.a.aO.9.A.A&,[][a.A.b.b.h.h.j.j.p.p.r.rO1@1I1I1K1K.JPQX/ZlJmp  AS  qT  KTFJ'9;'FG !'*J!&,"3"3L"A"AB:NWWYYL%hn&9&9,&G&GHT]]__N*c1L'BH\,B,B+BCL##%% ZgZZZblmmm3z??++ + +!!!!Q$''5577 +)/9 ?)7)<)<)>)>'-7 ;'3'8'8':':+,N1a4()*LA&#L1L4NNF!&&((G"LGMgm00>>?HHJJQQRWXXG=* )!,,..aaag&Gggs:77Gl+GJvs9999Dvd||s:..
67;;;5<?##d** $==??) 7 7AIeAhmQ.///36////LLFEFFFjLYYYcEk1--I%OOEzzi1nn199 :&&&%/&&&8$$$-$$$$]] :&&&#	/9&&&8$$$	1$$$ nhzXJ/F&G3&N%X%X%Z%Z%^%^de%^%f%fgG! 1 1fl** %++C000#$$q((Y\_`p\q\qq  zD  E  E  E ! !fl** #Z_
u$$#);v|/C/CC/H/H1/L#ML%k#$$/GVC[
'-jC.?@IIKK
Z''-7-?EIc6k*J&&,6u,=EIc5j)Z''5((SGM!4D-D-D1=g1F	#v+.03EIc6k4JEIVY[`V`La0b0b	#u*--1;F1C	#v+.j((.8.AEIc7l+2<[2IEIc;.// J..2<[2IEIc;./z))/9(/C/I/I/K/K/R/RSX/Y/YEIc8m,.2	#{*+1C!J r#   c                 <   |dvr||fS |}|dk    rd}d}n|dk    rd}d}n
|dk    rd}d}|d	         d
k    }|                                 s||fS |j        t          j        |          d
         d                  }d}|j        t
          j                            d          t          j	        d          z
  k    rU	 | j
        d         |d         z  }t          ||z  dz
            dk     rd}n"# t          $ r t          j        j        s Y nw xY w|rt           D ]}	||	xx         |z  cc<   || j
        d<   |d         dk    }
|
                                 r|ddg                                         }|d                                                             d|d         j        d
                   |d<   ||
         }|d         |d         z                                  }t+          |          d
k    r(t          j        |          dk    r|dxx         |z  cc<   ||fS )N)GBpZAcILAr  GBPru  r  ZARr  ILSri   r   rN   Tr1   rF   r3   regularMarketPricerg   r/   皙?Frb   rq   rw   rq  )r   r   r   r  r   r   r   r   r   r   r   r  r   r   r   r   r	   r~  r  shiftr|  r   r  )r   r   rb   	currency2rP   f_volumelast_rowprices_in_subunitsr  r   f_divr-  div_pctss                r!   r   z"PriceHistory._standardise_currency  sm   000x<	uIAAIAAIA h<>||~~ 	 x<728H--a045!=BL,,U33i6Ir6R6R6RRSS	./CDxPWGXXa{##c))).&   ~5 	
  	%  1
-6z*;$99;; 	% w{+,1133D M//1177d7mFXYZF[7\\DM;D[)DM9CCEEH8}}q  RZ%9%9A%=%=;1$9}s   ;1C- -DDc                    fd|d                                          D             }g d}|D ]}d }|dk    r d}d}	n,dk    r d}d}	n||v r|v r
|  d}d}	n| d}d}	 d}t          | j        || j                  }
|
                    d|	          d
         j        d         }|	rd|z  }|j        |d         |k    dfxx         |z  cc<   |dt          | j        || j                  }|                    d|	          d
         j        d         }|j        |d         |k    dfxx         |z  cc<   |d<   |S )Nc                      g | ]
}|k    |S r   r   )r   r   fxs     r!   r   z6PriceHistory._dividends_convert_fx.<locals>.<listcomp>>  s    SSSA1PR77a777r#   rb   )USDJPYEURCNYr  CADr  =XFTr*   )r   rx   rg   rN   r/   rq   )uniquer   r   r   r  r   r   )r   r
  r  rx   bad_div_currenciesmajor_currenciesr   fx2_tkrfx_tkrreversefx_datfx_ratefx2_datfx2_rates     `           r!   r   z"PriceHistory._dividends_convert_fx=  s   SSSS:)>)E)E)G)GSSSEEE# 	Q 	QAGEzzu&&&21A+A+Ar
 )))!$*fdlCCFnnE&nAA'JOPRSG $G)M)J/2K?@@@GK@@@"&tz7DLII"??%?GGPUVXYi
3Q6CDDDPDDD "	*r#   c                 v    |j         r|S |                     |||          }|                     ||||          }|S N)r   _fix_unit_switch_fix_unit_random_mixups)r   r   rA   r  r   rH  df3s          r!   r   zPriceHistory._fix_unit_mixupsd  sF    8 	I##B+>>**3+wOO
r#   c                   # |j         r|S t          j                    }d|| j        d}|j        d         dk    rd|j        vrd|d<   |S |j        d         dk    r'|                    d|           d|j        vrd|d<   |S |                                ##j        j	         #j        
                    |          #_        n/#j        j	        |k    r#j                            |          #_        dd	lm} g d
}#fd|D             }#|         dk                        d                                          }	|	                                r#|	         }
#|	          #||	          }nd }
|}#j        d         dk    r'|                    d|           d|j        vrd|d<   |S #|                                         }|                    |dd          }||z  }|dz                                  dz  }|dk    }d|z  }|dz                                  dz  }|dk    |dk    z  }||z  }|                                s'|                    d|           d|j        vrd|d<   |S d}t'          t)          |                    D ]"}|d d |f         }||         }|#j        ||f<   #||k                                    }|                     #|||          ##|                                         |k    }#|                                         |k                                    }|dk    r#|                                         |k    |z  }t'          |j        d                   D ]}||d d f         }|                                s$#j        |         }dD ];}|                    |          }||         r|j        ||f         dz  #j        ||f<   <d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   	#|                                         |k    |z  }t'          |j        d                   D ]}||d d f         }|                                s$#j        |         }dD ];}|                    |          }||         r|j        ||f         dz  #j        ||f<   <d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   	#|                                         |k    }|                                }n|}||z
  }||z
  } |dk    r/d| d| d}!| dk    r	|!d |  d!z  }!|                    |!|           #|                                         |k    }t'          t)          |                    D ]C}|d d |f         }"|"                                r!||         }|j        |"|f         #j        |"|f<   D|
Td|
j        vrd|
d<   t5          j        #|
g                                          #t5          j        #j                  #_        #S )"Nprice-repair-100xrJ  r   r~   Fr/   z3Cannot check single-row table for 100x price errorsrX  )ndimage)re   rd   rf   rg   rh   c                 &    g | ]}|j         v |S r   r   r   r   rH  s     r!   r   z8PriceHistory._fix_unit_random_mixups.<locals>.<listcomp>  s%    >>>1Q#+-=-=Q-=-=-=r#   ro   z6Insufficient good data for detecting 100x price errors)   r  wrap)r  mode   rv  r9  zNo sporadic 100x errors      )rd   rg   ru  re   rd   rg   rf         Y@zfixed /z currency unit mixups (z	 crudely))r   r   r   r   r   r   r   r~  rj   r   r   r   scipyr  r   r|  r{  median_filterr   r@   r   r   r   r  r5   r   r   r   r   r   )$r   r   rA   r  r   r   r  _ndimager  f_zeroes
df2_zeroesdf_origdf2_datamedianr  ratio_roundedr  r  ratio_rcp_roundedf_rcpf_eitherr  rS  fir   n_before
df2_taggedn_afterr  r  n_after_cruder  n_fixed_crudely
report_msgfjrH  s$                                      @r!   r  z$PriceHistory._fix_unit_random_mixupsl  s    8 	I $&& 3H[_[fgg
8A;!"*,,"';I8A;!LLNV`Laaa"*,,"';Iggii9<	--k::CIIY\[((	,,[99CI 	.-----AAA	>>>>	>>>		Na',,!,44==??<<>> 	XJxi.C(mGGJG9Q<1KKPXbKccc"*,,"';Iy>**,,''vF'KK6!**,,r1S I	&^2244r9#%*;s*BCu9||~~ 	LL2*LEEE"*,,"';I s9~~&& 	! 	!A!!!Q$B!A CGBENNO((**//XwLL^,,..#5
y>**,,388::Q;; Y((**c1Q6A171:&& L Lq!!!tWvvxx il* @ @A!**A!u @*,&a.4*?QOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFOOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFO^,,..#5>E5;q>** L L1aaa4[vvxx il* A AA!**A!u A*,&a.5*@QOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFOOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFOY0022c9J&NN,,MM#M]*!M1Q;;L'LLHLLLJ""</<<<<
KK
*K555 y>**,,3s9~~&& 	4 	4A!!!Q$Bvvxx 4aL!(RU!3A!*"444*/
;')S*-..99;;Csy11CI
r#   c                 b    | j         d         dk    rd}nd}|                     ||||d          S )Nrb   KWF  rv  T)correct_dividend)r   _fix_prices_sudden_change)r   r   rA   r  r  s        r!   r  zPriceHistory._fix_unit_switch  sC     !*-66AAA--b(K]a-bbbr#   c                 ,  ) |j         r|S t          j                    }d|| j        d}|d         dv }|                                }|                                ))j        j         )j                            |          )_        n/)j        j        |k    r)j        	                    |          )_        )fdt          D             })|         dk    )|                                         z  }	d }
|rt          j        |	                    d          d	
                              |	j        j                  }|                                |                                z  }|j        |dk             }t'          j        |	j        j        |          })|         }
)|          ))j         rd|j        vrd|d<   |S )                                ))|         dk    )|                                         z  }	)d                                         )d                                         k    }| j                            d          rd }n)d                                                                          )d                                                                          z  })d         dk                                    }||z  |z  }|sa)d                                         }d|j        d<   t'          j        |)d         z            }|dk                                    |z  }||z  }d)j        v rQ)d         dk                                    }|                                r|| z  }|                                rd|	|<   |	                                }	|	                    d          }|||z  }|                                s'|                    d|           d|j        vrd|d<   |S |	                                t9          |          t9          )          z  k    r'|                    d|           d|j        vrd|d<   |S |dgz   }d}t;          t9          |                    D ] }||         }|)j        |	d d |f         |f<   !||)j        |df<   )d                                         dk    )d                                                                         z  }|)j        |	                    d          |z  df<   |)j        ||z  df<   )|                                         |k    }|                                })j        |                    d                   } |                     )|||          ))|                                         |k    }|                                }!)j        |                    d                   }"||!z
  }#|#dk    rt| j         d|# d| d| d}$|#dk     rDtA          tC          tE          |           #                    |"                              }%|$d |% z  }$|                    |$|           |
6d|
j        vrd|
d<   t          j$        )|
g                                          ))|                                         |k    }&t;          t9          |                    D ]C}'|&d d |'f         }(|(                                r!||'         }|j        |(|f         )j        |(|f<   D)S )!Nzprice-repair-zeroesrJ  rN   rO   c                 &    g | ]}|j         v |S r   r   r  s     r!   r   z,PriceHistory._fix_zeroes.<locals>.<listcomp>.  s%    FFFAQ#+5E5Ea5E5E5Er#   rw   r/   ro   r  )r         ?r~   Fre   rf   r  ri   r   rg   皙?rm   TzNo price=0 errors to repairrX  z;No good data for calibration so cannot fix price=0 bad datar  z: fixed r  z value=0 errors in z price datar2   r.   )%r   r   r   r   r   r~  rj   r   r   r   r	   r   r   r   r   r  r   r   countr   r  r   r|  endswithrp  r   r  r   r   r@   r   r  sortedr   set
differencer   )*r   r   rA   r  r   r   r  r   r  f_prices_baddf2_reservegrpnan_pctdtsf_zero_or_nan_ignoref_change	f_vol_badf_high_low_good
f_vol_zero
close_diffclose_chg_pct_absf_bad_price_chgf_splitf_change_expected_but_missing
f_bad_rowsr  r  rS  r   f_vol_zero_or_nanr%  r$  
dts_taggedr&  dts_not_repairedr  r  dts_repairedr  r  r*  rH  s*                                            @r!   r   zPriceHistory._fix_zeroes  s!    8 	I$&& 5h]a]hii
B<:-]]__ggii9<	--k::CIIY\[((	,,[99CIFFFF!1FFF
J3.#j/2F2F2H2HH 	M)L,,!,445AAAII,J\JabbCggii#))++-G-#.C#%7<+=+BC#H#H 23K++,Cy bj00&+B{O	((**C
Os2c*o6J6J6L6LLLv;''))SZ-@-@-B-BB;%% 	8II #F 0 0 2 2 ; ; = ==3u:??CTCTC]C]C_C_B_`Oh-1,6688J"_4x?I  8 !\..00
%&
"$&F:G+D$E$E!#4t#;"E"E"G"G*"T%7	 S[((>*c1;;==G{{}} G078)0C-04466 GBFL!>? $,,..!%%1%--
 #i/J~~ 	LL6jLIII"*,,"';IZ3s88!;;;LLV^hLiii"*,,"';I(+	 s:'' 	1 	1A1A-0CGLA&)** +.CGIx'( ]3355:s8}?Q?Q?S?S?\?\?^?^_JM  a ((+<<hFG:=,,h67^,,..#5
>>##Yz~~1~556
//XwLL^,,..#5
..""9Z^^^%;%;<W$Q;;[ff'ffHffQYfffC{{%d3z??+E+EFV+W+W&X&XYY*L***LLJL///"+"555+0K()S+.//::<<C 	N##%%,s9~~&& 	/ 	/A111a4Bvvxx /aL!#AA
r#   c           
      	   d|j         vr|S |d         dk                                    r|S d}t          j                    }d| j        d}|                                }|                                }|d                             d                                           |d<   |d	         dk    |d         dk    z  }|j	        |df         
                                }|                    dd
          }d|j         vrd|d<   |d         |d         z  |d<   |rXt          j        |d<   t          j        |d<   |d         |d         z                      d          |d<   t          d|d           ||d         dk             j        }|d                                         }|d                                         }	i }
|D ]_}|j                            |          }|dk    r;|d	         j        |         }|d         j        |         }||k     rQ|||d
z
           z  }|||d
z
           z  }||d
z
           ||         z
  ||d
z
           z  }||z
  }t          ||z
            }t          |||z   z
            }||k     }||
|<   |rt          d|                                 d|dz  dd|dz  dd           t          d|dz  dd           t          d|dz  dd           t          d|d           t          d|d           t          d|            at+          |
                                          t/          |
          z  }|rt          d |dz  dd           |d!k    r|
                                D ]}|j        |         }|d	         j        |         }|d         j        |         }||z
  }||j	        |d	f<   |	|d
z
           ||d
z
           z  |	|         ||         z  z  }d"||z   ||d
z
           z  z
  }||z  }|j	        d |t3          j        d
          z
  dfxx         |z  cc<   d#|j	        d |df<   d$|                                 d%|d}|                    ||&           |r||j	        |df<   |d         |d         z  |d<   |r|d                             d          |d<   n|                    dd
          }|S )'Nrr   r   Fzrepair-capital-gains)rK  rM  rg   )fill_methodzPrice_Change%rq   r/   ro   r~   rh   AdjScaleFactor
correctionr2   AdjYahooz# price_drop_pct_mean = .4fz# z: div = rv  .1fz%, cg = %z- price_drop_pct = z- price_drop_pct_excl_vol = z- diff_div = z- diff_total = z- cg_is_double_counted = z- pct_double_counted = gZd;O?r9  Tz'Repaired capital-gains double-count at z. Adj correction = rX  )r   r   r   r   r   r~  r   
pct_changer  r   meanr   r   r  r   printrj   r|  r  r   r   r   valuesr   r  r   r   r{  )r   r   r   r   r  no_distributionsprice_drop_pct_meanr=  r   acdcsr  r  dividendr  div_pctcg_pctprice_drop_pctprice_drop_pct_excl_voldiff_div
diff_totalcg_is_double_countedpct_double_counteddividend_true
adj_beforeadj_correctrQ  r  s                               r!   r   z"PriceHistory._repair_capital_gains  s   
 "*,,I1$))++ 	I $&& 6T[QQ
WWYY]]__
 !k444FFJJLL?{Oq0R5HA5MN f%5%FGLLNNWW_1W-- bj((#B{O{Obk15	 	H "B}!vB| obk9@@CCBzNF-@FFFGGGO$q()/wK  ""_%%'' 	N 	NB(""2&&CQww k?/4 "? 3 8 =m++"Qs1uX-&3q51 #$CE(QsV"3qQx!?*8;N*N'6@AA !8Gf<L!MNN
'/*'<$/C N\rwwyy\\'#+\\\6RU:\\\\]]]Is0BIIIIJJJ[9PQT9T[[[[\\\8(888999<J<<<===L6JLLMMM ..S9 	KI,>s,BIIIIJJJ&&xxzz : :Xc]k?/4 "? 3 8 =
 !)= 8*7r;' !Qi#a%0RWQsV^D
!]]%BaAh$NN(:5
19.q111158999ZG999+/ssK'(n		nn^hnnnCz222 :/9BF2|+,W+5	1; 	(5	**BuIIQ''B	r#   c           	      6Y   ||j         r|S |dv r|S d|j        v r |d         dk                                    r|S t          j                    }d|| j        d}|d         dk                                    }|                                s|                    d|	           |S | j        d
         dk    rd}nd}d d}|	                                }|
                                }	d|	j        vrd|	d<   d}
|	d                                                                         }|	|         
                                }|	|          
                                }	|	d         dk                                    }|                                s|                    d|	           |S t          j        |          d         }g }t          t          |          dz
  dd          D ]@}||         }|dk    r|	j        |dz
           }|d         |d         z
  }|	d         j        |         }|dk    r||z  dz
  dk     r|	j        |dz
           }|d         |z   }||d         k    r||d         k    r||	j        |df<   |	d         j        |         |	d         j        |         z  }|d||	d         j        |dz
           z  z
  z  }|	d         j        |dz
           |z  |	j        |df<   d|	j        |df<   d}
|                    |	j        |                                                    Bt          |          dk    r&dd |D              }|                    ||	           t          t          |          dz
  dd          D ]K}||         }|	j        |         }|	d         j        |         }|dk    r2||	d         j        |dz
           z  }d}d}t-          |	d         j        |         |	d         j        |dz
           dz  d          r|	d         j        |dz
           dz  |	d         j        |         z
  }||	d         j        |dz
           dz  z  }d||	d         j        |dz
           dz  z  z
  }|	d         j        |dz
           |	d         j        |dz
           z  }t-          ||d          st||	d         j        d |t/          j        d          z
           z  |	j        d |t/          j        d          z
  df<   d|	j        d |t/          j        d          z
  df<   nt-          |	d         j        |         |	d         j        |dz
           dz  d          r|	d         j        |dz
           dz  |	d         j        |         z
  }||	d         j        |dz
           dz  z  }d||	d         j        |dz
           dz  z  z
  }|	d         j        |dz
           |	d         j        |dz
           z  }t-          ||d          st||	d         j        d |t/          j        d          z
           z  |	j        d |t/          j        d          z
  df<   d|	j        d |t/          j        d          z
  df<   n*|	d         j        |dz
           |	d         j        |         z
  }|} |t          |	          dz
  k     rSt          j        | |	d         j        |         |	d         j        |dz            z
  g          }!t          j        |!          }"nt          j        | g          }!| }"t          |	          |z
  d k     r4t7          t          |	          |d z             }#t5          d|#d!z
            }$n3t5          d|d z
            }$t7          t          |	          |$d!z             }#|#|$z
  d k     rt          j        }%n{|	d         j        |$|#dz
                                           |	d         j        |$dz   |#                                         z
  }&t          j        t          j        |&                    }%g }'|!dk                                    r|	d"         j        |         dk    r}t          j         |	d"         dk              t          |	          z  }(|dz  d#k     r|'                    d$dd%           n<|(d&k    r|d'k    s|dk    r|'                    d(dd%           n|	d)         j        |         })|)dk    rd }*n^||)z  }*|*|k    r| |%z
  }+n|"}+|+dk    rEt=          ||+z
            }t=          |*|+z
            },|,|z  |k    r|'                    d*|,d%           ||k    r|"dk    r|'                    d(dd%           nt=          ||"z
            }t=          ||z  |"z
            }-|*"|-|z  |k    r|'                    d(|-d%           n^t=          |*|z  |"z
            }.|-|.k     r"|-|z  |k    r|'                    d(|-d%           n!|.|z  |k    r|'                    d+|.d%           t          j!        |%          s| |%z
  }/|/dk    rt=          ||/z
            }t=          ||z  |/z
            }-|*"|-|z  |k    r|'                    d$|-d%           n^t=          |*|z  |/z
            }.|-|.k     r"|-|z  |k    r|'                    d$|-d%           n!|.|z  |k    r|'                    d,|.d%           ||||d-}0| |0d.<   |"|0d/<   |	d"         j        |         |0d0<   |%|0d1<   d|0d2<   d|0d3<   d|0d4<   d|0d5<   d|0d6<   t          |'          dk    r9tE          |'d7 8          }'|'d         }1d|0|1d9         #                    d:d;          <   tI          j%        |0g          &                    d<          5tI          j'        g          M|
s|S d= j        D             }2	                                dd@}3t          t                              D ]}dA         j        |         }j        |         }d>         j        |         }|dk    r=||	d         j        |dz
           z  }|	d         j        |dz
           |	d         j        |dz
           z  }4|	d         j        |         |	d         j        |         z  }5|5|4k    }6|4|5z  }d|z
  }7|7d#|z  k     }8|7dB|z  k    }9|6rd}8dC|i}0|6|0dD<   |9|0dE<   |8|0dF<   |0(                                D ]\  }:};|:vr{tS          |;tT          t          j+        f          rd|:<   nTtS          |;tX                    rd|:<   n9tS          |;tZ                    rd|:<   nt]          |:|;t_          |;                    |;j        ||:f<   |2g dGz  }2ddH<   t/          j        dIJ          }<d2dFg                             dK          }=|=                                rTt                    dk    r@t          j        |=          d         }>j        |=         }?|>D ]}j        |         }|j0        }@d }A|dk    rj        |dz
           }Bnj        |dz            }B|d>         |z  |Bd>         z  }C|d>         |Bd>         z  }Dt7          t=          |Cdz
            t=          |Ddz
                      }Et=          |@|Bj0        z
            |<k    r=|BdH         s5|Edk     r/|Bj0        |?v r$|d.         } |Bd.         }F| dL|Fz  k    r|Bj0        }An|@}An|@}A|Ar&dj        |AdHf<   |2D ]}G|Gj        v rdj        |A|Gf<   dM}H	                                t          dt                              D ]}j        |         }|j0        }@d}Id}J|d.         } j        |dz
           }K|d>         |Kd>         z  }Lt=          |@|Kj0        z
            |<k    rJ|KdH         sB|dH         s:t=          |Ldz
            |Hk     r$d.         j        |dz
           }M| dL|Mz  k    rd}Jnd}I|Js|Ir:|Ir|@nj        |dz
           }Ndj        |NdHf<   |2D ]}G|Gj        v rdj        |N|Gf<   |2                    dH           dHj        v rrdH         }O|OdN |2D                                          dK           z  }O|O                                rj        |O         }PdOdP |PD              }|                    ||	           j        |O         }Q|QD ]}|t/          j        d          z
  }R|	j        d |Rdfxx         dC         j        |         z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         dC         j        |         z  cc<   d|j        d |Rdf<   d|	j        |df<   d}
1                    |          dj        |OdHf<   1                    dHdK          dH|2v r|22                    dH           |2                                                                         st                    dk    rt          dt                              D ]}dQ\  }S}T|dk    r*dR         j        |dz
           dR         j        |         z  }S|t                    dz
  k     r*dR         j        |dz            dR         j        |         z  }T|Sp|T}S|Tp|S}Tt=          |S|z
            dSk     r/t=          |T|z
            dSk     rj        |         }@dj        |@d3f<   ŉ|2                                                                         s5|
r1|j         s(tI          j'        |	|g          	                                }	|	S |S t          t                              D ]o}dA         j        |         }j        |         }d>         j        |         }|dk    r=||	d         j        |dz
           z  }|t/          j        dTJ          z   }Utg          j4        |	j        |U          }Vt7          |Vt          |	          dz
            }Vt5          d|dUz
            }W|	d         j        ||Vdz            5                                }X|Xdk                                    |XdVk                                     z  }Y|Y                                r3|t          j        |Y          d         d         z   dz
  }V|	j        |V         }Ud}Zd}[tH          j6        }\|V|Wk    r5|	j        |W|Vdz            
                                }]|]d         |]d         z  |]dW<   |]dW         |]d         z  |]dX<   |]d         j        dd                                          |]d         j        d d                                         z
  }^t          j        dg|^          }^|^|]dY<   |]dX         j        dd                                          |]d         j        d d                                         z
  }_t          j        dg|_          }_|_|]dZ<   |]dYdZg         }^|d[k    r|dk     r||]dW         j        d         z  }`|^dZ         |`d\z  k    }=|=                                rt          j        |=          d         }>|>D ]}a|^dZ         j        |a         }b|bd]|^dY         j        |a         z  k    rtd|^dZ         z  |bz  }c|cd^k    }d|	d)         j        |         })d4         j        |         }e|)dk    s|es|d                                rt          j        |d          d         }ft          |f          dk    rt          j7        t          |f                    }gt          t          |f                    D ]Q}|f|         }a|cj        |a         }h|h|k     r|g|xx         ||hz
  j8        z  cc<   6|g|xx         d#|h|z
  j8        z  z  cc<   Rt          j9        |g          }|f|         }in|fd         }id}[|cj        |i         }\ n-|bd_|`z  k    r"|]dW         dk                                     rd}Z nj        |dFf         }8|Zr|8rd}Zi }0|Z|0d`<   |[|0da<   |\|0db<   |Zr|	d)         j        |         })|)dk    r||)z  }*|*|k    r,d1         j        |         }%d.         j        |         } | |%z
  }+nd/         j        |         }"|"}+|+dk    r9t=          ||+z
            }t=          |*|+z
            },|,|dcz  k    rdj        |d4f<   |0(                                D ]\  }:};|:vrtS          |;tT          t          j+        f          rd|:<   nstS          |;tX                    rd|:<   nXtS          |;tZ                    rd|:<   n=|:dbk    rtI          j:        dde          |:<   nt]          |:|;t_          |;                    |;j        ||:f<   d2j        v r-daj        v r$dj        da                                         d2f<   q|2d`dagz  }2|2D ]3}G|G                                         s1                    |GdK          4db}G|Gj        v rC|G                                                                         r1                    |GdK          fdf|2D             }2;                    dR           |3dRg          dh<   dh         <                                }j|jD ]}kdh         |kk    }l|l         	                                t                    }mdRg         
                                }nt          |n          dk    r|ndR         j        =                                >                                j?        @                                diz  }o|oj        d         |oj        |oj        d         <   |o|ndj<   |ndR         |ndj         z  |ndk<   |2D ]E}G|G                                                                         s1                    |GdK          Ffdl|2D             }p|pD ]}G|G                                         }qt          j         |q          }r|rd|mfv r8|r|mz  }s|Gd2k    rd}td'}udaj        v r%|G         da         k                                    rvd`j        v r|G         |G         d`         z  k                                    rdm}td`                                         }vt          j         |v          }mt          j         |q|v                   }r|r|mz  }s|s|tk    r)|ld`                                         z  }=dj        |=|Gf<   ,dFj        v rJdF                                         r0d1         |l         |q         dk                                    rdn}un5d'}tn2d`j        v r'|G         d`         k                                    rdm}tndm}u|s|tk    rSdj        |l|Gf<   daj        v r<dj        |ldaf<   tH          j6        j        |ldbf<   |l         	                                
|s|uk    rdj        |l|Gf<   |Gd3k    r9d}td#}udEj        vrdo}tdm}u|s|tk    rdj        |l|Gf<   I|s|uk    rdj        |l|Gf<   ]|GdDk    r|G         j        d         r|rdk    r~|GdFk    r|Gd`k    r|GdHk    r| j        dpv r|Gdak    r|Gdqv rd2|2v rdF|2v rdr}Gd|G<   t          t                              D ]}}j        |         }j        |         d2         rYdF         rQd>         }wdR         }x|w|xz  }ydC         }zd|zz
  |yz  }7|w|7z  }Lt=          |L||z  z
            |k     }{|{rdj        ||Gf<   ~|G                                         s1                    |GdK          n|2                    |G           d5j        v rjdsD ];}G|Gv r|G         d5         z  |G<   d5         |G<   |2                    |G           <1                    d5dK          |22                    d5           	                                |2                             dK                   j         r1|j         s(tI          j'        |	|g          	                                }	|	S i }|t          dh         <                                          D ]^}kdh         |kk             	                    dt          ddu<   t          t                    dz
  dd          D ]}j        |         j0        }|t/          j        d          z
  }RdDv odD         }}dFv odF         }~dEv odE         }d`v od`         }d3v od3         }d2v od2         }d4v od4         }dav oda         }[drv odr         }t          j         fdv|2D                       }|r|r|dk    rd}|dz  }|[r*|rd}dj        |d2f<   |dz  }|~rd}~dj        |dFf<   |dz  }|r|rd}dj        |d`f<   |dz  }|dk    r|~s|r|~rdwndx}:||B                    |:g                               |           ddR         z
  dC         z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   |rdy}:|}d>         |z  }||	j        |df<   |:dzz  }:dddC         z
  |z  z
  }|dC         z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   ||B                    |:g                               |           ||rd{}:d|z  }d>         |z  }||	j        |df<   dR         |z  }d|z
  }dC         }|:dzz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   ||B                    |:g                               |           C|}rd|}:||B                    |:g                               |           ddR         z
  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   |[r0d}}:||B                    |:g                               |           ddC         z  }|	j        d |Rdfxx         |z  cc<   |j        d |Rdfxx         |z  cc<   db         }\|	d         j        dA                  }d>         }d||z  z
  }|\t/          j        d          z
  }|	j        d |dfxx         |z  cc<   |j        d |dfxx         |z  cc<   |	j        |\dfxx         |z  cc<   d|	j        |df<   d|	j        d |Rdf<   d|j        d |Rdf<   dj        |duf<   
|rddR         z
  }dC         }t=          ||z  dz
            d[k    rf|:dzz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   1                    |          1                    |          |rd~}:d|	d)         j        |         z  }d>         |z  }||	j        |df<   dR         |z  }d|z
  }dC         }|:dzz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   ||B                    |:g                               |           |dk    r|r|}rd}:||B                    |:g                               |           ddR         |z  z
  }|	j        |dfxx         |z  cc<   |	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   Y|rm|~rkd>         }|dR         z  }ydC         }z|	j        |dfxx         |z  cc<   d(}:||B                    |:g                               |           dj        |duf<   |r|rd}:||B                    |:g                               |           dR         |z  }d|z
  }|dC         z  }|	j        |dfxx         |z  cc<   |	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   |r|r|	j        |dfxx         |z  cc<   dy}:dv rvd         rnd }ddR         |z  z
  }|dC         z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   |:dz  }:||B                    |:g                               |           dj        |duf<   	S|dk    r|r|~r|rd}:d|z  d|	d)         j        |         z  z  }d>         |z  }||	j        |df<   dR         |z  }d|z
  }dC         }|:dzz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |duf<   ||B                    |:g                               |           
<|r|~r|rd>         }|dR         z  }ydC         }zd}:||z  }||yz  }||	j        |df<   d|z
  }||zz  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   ||B                    |:g                               |           dj        |duf<   j         r^`||D ]>}:d|: dd tE          |||:                   D              }|                    ||	           ?|j         s(tI          j'        |	|g          	                                }	|	S )N)r,   r*   r-   1yrr   r   zdiv-adjust-repair-badrJ  rq   rw   zNo dividends to checkrX  rb   r,  r-  rv  gQ?r~   Frg   r/   rN   rf   ru  re   rh   r9  Tz'Repaired double-adjustment on div days c                 ,    g | ]}t          |          S r   )r   )r   rN  s     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>^  s    <Y<Y<YSVV<Y<Y<Yr#   rZ  g?)rel_tol)secondsr2      ri   r  zdiv-too-small)staterp  g      ?g      ?zdiv-too-bigrm   zdiv-pre-splitzdiv-too-big-and-pre-splitzdiv-too-small-and-pre-split)r   r  divrU  r   
drop_2Dmaxvolumevoldiv_too_bigdiv_too_smalldiv_pre_splitdiv_too_big_and_pre_splitdiv_too_small_and_pre_splitc                     | d         S )Nrp  r   )r   s    r!   <lambda>z2PriceHistory._fix_bad_div_adjust.<locals>.<lambda>  s
    AfI r#   )keyrp  -_r   c                 <    g | ]}|                     d           |S )div_)
startswithr   r   s     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s)    KKKall66J6JK!KKKr#   rq  rk  c                    t          |           }|                     |          }g }|j        d         g}||         j        d         g}t	          d|          D ]}|j        |         }	||         j        |         }
|
t          j        |          z  |k     r+|                    |	           |                    |
           h|                    |           |	g}|
g}|                    |           t          j        dg|z            }d}t          |          D ]#\  }}t          |          }|||||z   <   ||z  }$|S )Nr   r/   rN   )
r   sort_valuesrj   r   r@   r   rW  r  array	enumerate)r   column	thresholdr  	sorted_dfclusterscurrent_dtscurrents_valsrS  r  rq  cluster_labelsrc  clusterncs                  r!   cluster_dividendsz;PriceHistory._fix_bad_div_adjust.<locals>.cluster_dividends  sZ   BAv..IH$?1-.K&v.3A67M1a[[ * *_Q'',Q/"'-000I==&&r***!((---- OOK000#%$K%(EMMOOK(((Xrd1f--NC'11  
7\\-.s3r6z*r	!!r#   r  r%   zpresent adjadj_missingadj_exceeds_divdiv_exceeds_adj)r  r  r  phantom   r3   ro         ?g{Gz?c                     g | ]
}|d k    |S )r  r   r  s     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s    4Y4Y4Y1!y..Q...r#   zRemoving phantom div(s): c                 P    g | ]#}t          |                                          $S r   r   r   r   r  s     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s&    2T2T2Tb3rwwyy>>2T2T2Tr#   )NNrU  r  #      grO  zAdj LowdeltaadjDeltar3  g333333?gjt?皙?g(\?adj_exceeds_pricesdiv_date_wrongdiv_true_dateg?zdatetime64[ns, UTC])dtypec                 &    g | ]}|j         v |S r   r   )r   r   div_status_dfs     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>P  s&    BBBqM,A'A'A!'A'A'Ar#   )r  r  g    ~~Ar   zavg yr yieldc                 &    g | ]}|j         v |S r   r   )r   r   r  s     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>h  s%    HHHA13G3Ga3G3G3Gr#   r2  gUUUUUU?gtE]t?)zKAP.ILSAND)rw  rx  adj_too_small)ru  rw  	ascendingzFixed?c                 (    g | ]}|v |         S r   r   )r   r   rows     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>	  s"    )M)M)MQAHH#a&HHHr#   ztoo-small div-adjustztoo-big div-adjustztoo-small divz & div-adjustztoo-big divzmissing div-adjustzwrong ex-div datezpre-split divz"too-big div and missing div-adjustztoo-big div & div-adjustzFX was repairedz and FX mixupr  ztoo-big div & pre-splitzdiv-too-big and adj-too-smallz	Repaired r.   c                 P    g | ]#}t          |                                          $S r   r  r  s     r!   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>
  s&    #T#T#TrC		NN#T#T#Tr#   )rq  rk  )Cr   r   r   r   r   r   r|  r   r   r   r~  r   r   r  r@   r   r   rj   r   r  r   r{  r   r   r   r  r5   r   r  rW  r  r   r   r  r6  replacer   r   	set_indexr   itemsr   boolbool_r   r  r   rx  r   r   removebisectbisect_leftrV  NaTzerosr4   argminr   r  r  	to_seriesrp  r  total_secondsr   
setdefault)r   r   rA   rb   r   r  r  currency_dividetoo_big_check_thresholdrH  df_modifiedf_nandf2_nandiv_indicesfixed_datesrS  div_idxprices_beforerp  rq  	dt_before	new_close	adj_afteradjr  r  r_  #div_too_small_improvement_threshold!div_too_big_improvement_thresholddrop_c2ltrue_adjustpresent_adjr   dropsrr  r   r   typical_volatilitydiffspossibilitiespct_zero_volr   div_postSplit_dropdiff_postSplitdiff_fxdiff_fxPostSplitdrop_wo_vol
div_statuspchecksr  pre_adjpost_adjdiv_missing_from_adjcloseimplied_div_yielddiv_adj_is_too_smalldiv_adj_exceeds_divr   vphantom_proximity_thresholdr  indicesdts_to_checkdiv_dt
phantom_dt	other_divratio1ratio2
divergence	drop_nextr   ratio_thresholdthis_is_phantomlast_is_phantomlast_divr  	last_dropphantom_div_dt	f_phantomdiv_dtsphantom_div_dtsenddtr_prer_postlookahead_datelookahead_idxlookback_idxfuture_changesf_big_changediv_adj_exceeds_pricesr  r  xdeltas	adjDeltasadjDivr  adjDelta_dropr  f_near1_or_above	pre_splitnear_indices	penaltiesdtireversal_idxcluster_idscidfcr  r  
time_diffscluster_checksf_failn_failpct_failtrue_thresholdfals_thresholdf_adj_exceeds_prices	div_yieldpctcloseadj_presentalso_correct_adjdiv_repairsr  r  r  r  rv  ru  rw  r  n_failed_checksadj_correctionrQ  correct_div
target_adjtarget_div_pctclose_beforetrue_adjenddt2r  div_truepct_trueri  r  r  r  s                                                                                                                                                     @@@r!   r   z PriceHistory._fix_bad_div_adjust  s6    ::I222Ibj((b.A!.C-H-H-J-J( I$&& 7_c_jkk
KC'1133yy{{ 	LL0
LCCCI!*-66"OO!O"']]__ggiick))$C G!!##,,..e*//##5&k  [!S(2244yy{{ 	LL0
LCCCIhuooa( s;'')2r22 	B 	BA!!nG!||HWQY/M '-*@@Dk"'0CaxxT#XaZ--  Igai0	)'2S8	e 444mTZF[9[9[2;CGIw./ #K 0 5g >WARSZA[ [I#sSW1B7191M-M'MNC69'l6GPQ	6RUX6XCGI{236:CGI{23"&K&&sy'9'>'>'@'@AAA{a[<Y<Y[<Y<Y<Y[[CKK:K... s;'')2r22 ]	@ ]	@A!!nG7#Bk"'0C!||CL-gai88G 34/01-s5zw/W1B7191Mc1Q]bccc Sw<,WQY7;c%jog>VVW!2719!=c!AB!C3w<+<WQY+G+K$LL!+.3GAI>WARSZ[\S\A]]{K5III TOZ]`ah]i]m  oQoqr{  sF  OP  sQ  sQ  sQ  pQ  oQ  ^R  PRCG>R	 3A > > >>>KLOSCG>R	 3A > > >>>KLU13w<3DWQY3OPT3T`efff 
Sw<,WQY7<s5zw?WWW!2719!=d!BC!C3w<+<WQY+G+K$LL!+.3GAI>WARSZ[\S\A]]{K5III TOZ]`ah]i]m  oQoqr{  sF  OP  sQ  sQ  sQ  pQ  oQ  ^R  PRCG>R	 3A > > >>>KLOSCG>R	 3A > > >>>KLw<,WQY7#e*/':RRDS!## $G(9'(BSZ_U\]^U^E_(_!`aaVE]]

$((!
C A%%#c((GAI..As1uAwqy))#c((E!G,,5y1}}%'V""G)%A+6??AACJOTYZ[T[\_T_D`DiDiDkDkk%'WRVE]]%;%;"Ms
!! C{c(m&8&A1&D&D!vc(mS&899#c((B3;$$!((/#)N)NOOOO #T))gnn'C--!((-)L)LMMMN+/3C<<$(MM %(%KM$s** $'9 9 *qyy"3u9~~),]5-@)A)A*-NNSWWW)00/R`1a1abbb 444!S((%,,mC-P-PQQQQ"3z>22"%s?':J&F"G"G(0 '*K KPTTT - 4 4mT[5\5\ ] ] ]/2M/4QS]3]/^/^,&)999$+.O$OTX#X#X$1$8$8-X_9`9`$a$a$a$47X$X]a#a#a$1$8$8B]fv9w9w$x$x$x x 233 { #');";K"Q"3{?33"%s?':K&G"H"H(0 '*M MRVVV - 4 4oV]5^5^ _ _ _/2M/4QS^3^/_/_,&)999$+.O$OTX#X#X$1$8$8/Za9b9b$c$c$c$47X$X]a#a#a$1$8$8B_hx9y9y$z$z$z"$GCgNNJ!%Jv'1J|$#&x=#5g#>Jx  2Ju(-J}%*/J'*/J'6;J238=J45=!!A%% &}:M:M N N N!!$;?
1W:--c3778,
|,,66v>>C$ # "	=#*> ? ?  IKK]2KKK%0022	" 	" 	" 	"> s=))** ,	- ,	-A#E*/2G$Q'B&+A.C!||CL-gai88G +&+GAI6W9J7ST99UUG;',W5G8I'8RRH(0G(;% "H,K #k 1#4G#D  #4r'z"B ) -',$'5J(AJ}%,?J(),@J()!'')) - -!M))!!dBH%566 	6+0a((#As++ 6+,a((#Au-- 6+.a(( )1T!WW555+,!"a%((- 	GGGG#(i &/&9r&B&B&B#=*;<=AAqAII5577 	Es=))A--hqkk!nG(.q1L E E#(+!
q55 - 21Q3 7II - 21Q3 7Ie*_4	%8HHUi&66 VCZ#fSj//BB
vin,--1LLLU^_hUiLnx{nn ~55"6{$-f$5	#i-//)2JJ)/JJ%+
 E?CM%j)&;<# E E 555?DM-j!m<  %0022q#m,,-- 	E 	EA$Q'CXF#O#Ov;D$)!A#.HJ%0E6(-'((,GGGPXYbPcGlopylzG  @C  DI  JM  DM  @N  @N  Q`  @`  @`)&16qs;	#i-''&*OO&*O E/ E+:!X@STUVWTW@X?C!.)";< E EAM111?D).!*;<i    ---%i0I!m4Y4Y4Y4Y4Y&Z&^&^de&^&f&f%fgI}} @'-i8V2T2TG2T2T2TVVCz222"/"5i"@) ; ;By21====EG 3444m8T8XY[8\\4447;CG 34K 3444m8T8XY[8\\4447;GK 34/0CGBO,"&K$1$6$6r$:$:MM:?!)Y"67)..yq.AAMF""i(((V$((**..00 	J =!!A%%q#m"4"455 
J 
JA$.ME61uu -c 2 7! <}S?Q?VWX?Y YC..q011!.s!3!81!=c@R@WXY@Z!Z!OVE#_uF5011"44VO=S9T9TUW9W9W!.!4Q!7EI)&/*ABV$((**..00 	 } A)S'N33>>@@C
	 s=))** @	e @	eA#E*/2G$Q'B&+A.C!||CL-gai88G  	 3 < < <<N".sy.IIMs3xxz::Mq'"*--L !\.w}Q/FGRRTTN*Q.88::nt>S=]=]=_=__L!! : '"(<*@*@*CA*F Fq H!$=!9%*""NFM|++H\-/9:??AA[>AgJ6% x!E(2)5qrr*3355'
8L8U8U8W8WWC5&11#'
iL-abb1::<<q~?RSVTVSV?W?`?`?b?bb	IseY77	 )*GZ01T>>gmm 1U8=#33Fz*fSj9Auuww %*"$(1++a.#* #* #*C,2:,>,CC,HM,uVG_5I#5N/NNN +-VJ-?*?)N393; 0 ),N(;(?(C,9/,J,Nr,R	$)3JJyJGWG[G[G]G]J358<L3M3Ma3PL'*<'8'81'<'<46HS=N=N4O4O	16s<7H7H1I1I )R )RA2>q/C28,s2CC/2Rxx09!C0M09!SVM@Q0Q,.Ii,@,@7CA7CA59N4:L4NM$)E%2T&[%@%@ )*%'9'9';'; %FAE(>$)E $1#4R9J5J#K % /*> /).&J/EJ+,+9J'(*7J'% JN+/3C<< %(%KM$s**-:5-A-Eb-I*,V48< $'9 9%2<%@%DR%H
 *qyy"3u9~~),]5-@)A)A)d3h77EIM-b/.AB!'')) - -!M))!!dBH%566 	6+0a((#As++ 6+,a((#Au-- 6+.a((o--+-9;P+Q+Q+Qa(((1T!WW555+,!"a%(( 555:JmNc:c:c_d!-0@"A"J"J"L"Lm"[\')9:: 	> 	>A #'')) > - 2 211 2 = =%%%-*:*?*?*A*A*E*E*G*G%)..qq.99MBBBBVBBB
 &11#66#4#4]3#O#O#Oi  $I.5577 i	 i	Cy)S0B#B'2244GGAu~**,,H8}}q  %c]0::<<AACCFTTVVZop
6@oa6H
z/23%/"+3C=8H;M+M( 6 6qz**,,0022 6%ll11l55GHHHHHHHN# Y Y ,,..aV##!A:%%%(N%)N'7?::
gVfNg@g?l?l?n?n: +w>>GAJSZ[\S]`gh|`}S}D~  DD  DD  DF  DF> *-/67K/L/U/U/W/W,F#788!#/C(D!E!E#)A:#n44 "]3G%H%Q%Q%S%S SA6:M-ad3 (GO;;HY@Z@^@^@`@`; $EN2.v69>>@@ 2-0NN .2NN-@@gajRYZnRoFoEtEtEvEv@ *- *->1137)"a%0+}/DDDFKM-b2B.BCEGVM-b/.AB&3B&7&B&B&D&DG !^3338)"a%0 ''%(N%(N(??)-),>1137)"a%0 !^3338)"a%0 %%qzr* !v{{ ))),,,	>>dk5G&G&G(((FFF GoYv F""'8F'B'BA$M!3}--.. 8 8"(+#(+}% 
8#.?*@ 
8 #E
Ic(C%cME"%m"4K)*;(=%%&77E'*5//2Q+R'S'SVe'e$' 837)"a%0 #'')) ! - 2 211 2 = =a   &-*???5 % %%%'4Q'7-Hc:d'dM!$$'45P'QM!$MM!$$$$)../JQR.SSMMM5666%0022 &mF&;&?&?Q&?&G&GH 	= =ig//::<<J i07799:: S	 S	C#M)$<c$ABG((5(99G %GH3w<<>2r22 K9 K9l1oX9.q9999+s2Is=7I"3s":"UsCT?U"3s":"UsCT?U%9S%@%^SI]E^" /3 6 O3;O+s2Is=7I /3 6 O3;O "2S!8!RSAQ=R /3 6 O3;O"$&)M)M)M)M&)M)M)M"N"N )#5 )/Q:N:N */&#q(O! 
-" -&+9>B$56'1,& -*/=BB(9$9:'1,  -) --2*@EB(<$<='1,"a''& yA/ yA6E_22K_#..q"55<<R@@@*-C.C<N)NFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11& nA ,%4
&)%j:&=3>K0 _,%(S3}3E-E,S%T
)3c-6H)HFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@@$ [A)%(%8
&)%j:&=3>K0),SJ)>%(>%9
&)-&8_,)3k)AFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@@$ GA0#..q"55<<R@@@),SXFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11' =A/#..q"55<<R@@@ *-S-?)?FUFK$7888NJ888FUFK$7888NJ888 ),O(<'*7|'8U'D!%j#&\)9#9!.y/B1/M/M/M!MGVG[$8999XE999GVG[$8999XE999 { :;;;sB;;;34K0;?FUFK$78;?FUFK$7848BL11+ #A &)3s8^
&)-&8
; 69::TAA0A-7+-ENG(;<<<N<<<?CCG(;<#K(;<<<N<<<?CGK(;<8<GKH55,9,>,>r,B,BM&-ll2&6&6GG& A+%(^)<)@)D%D
&)%j:&=3>K0),SJ)>%(>%9
&)-&8_,)3k)AFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@$))" :9{ :9@#..q"55<<R@@@),s3x/G)GK000OC000FUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11$ .9 .9!%j #CH&)-&8 K000OC000)#..q"55<<R@@@48BL11$ #9); #9 7#..q"55<<R@@@),S/)A%(>%9
)3c-6H)HK000OC000FUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11& 9? 9 K000OC000+,33<M8N3 #'C&)SXo-E&FG-4s=7I-ING(;<<<N<<<?CCG(;<#K(;<<<N<<<?CGK(;<0A#..q"55<<R@@@48BL1$))" &9 &9= &95&)/&9c#nBUBYZ\B]>]%^
&)%j:&=3>K0),SJ)>%(>%9
&)-&8_,)3k)AFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@@$ 9 9] 9 "%j #CH&)-&8;#&#6#+e#33;K0&)Hn)4{)BFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$78#..q"55<<R@@@48BL1}   	/ 	/AVaVV#T#TVKPQN=S=S#T#T#TVVCKK:K....} 	9)S'N++6688C
r#   c           
         |j         r|S t          j                    }d|| j        d}|dv }|s|S |                                }|d                                         dk    }|                                s|                    d           |S |                    dt          |d         |         	                                           |           d	|j
        vrd
|d	<   t          j        |          d         D ]}|j        |         }	|j        |	df         }
|	|j        d         k    r1|dv r|dz  }n|dz  }t          |j        d         |          }|j        d|dz            }|                    d| d|	                                 d|
d|           |                    d|j        d                                          d|j        d                                          |           |                     ||||
dd          }||j        d         dz
  k    r|}*|j        |dz   d          }|j         r|                                }Yt)          j        |                                |g          }|S )Nzsplit-repairrJ  r$   r,   r*   r-   rm   r   z%price-repair-split: No splits in datazSplits: rX  r~   Fr,   r*   r-   r/   r(   z
split_idx=z
 split_dt=z split=rS  zdf dt range: rU   rN   T)correct_volumer.  )r   r   r   r   r   r|  r   r   r   to_dictr   r   r  rj   r   r   r   r   r   r/  r   r   )r   r   rA   r  r   r  interdaysplit_f	split_idxsplit_dtr   
cutoff_idxdf_pre_splitdf_pre_split_repaireddf_post_cutoffs                  r!   r   z"PriceHistory._fix_bad_stock_splits
  s    8 	I$&& .xVZVabb
:: 	I]]__^$--//14{{}} 	LL@AAAILB~$6w$?$G$G$I$I J JLLT^___bj((#B{O'**1- 	Y 	YIx	*HF8^34E28A;&& 000Q		Q	RXa[)44J71Z\>2LLL^i^^8==??^^SX^^^fpLqqqLLj);A)>)C)C)E)Ejj<K]^`KaKfKfKhKhjjr|L}}}$($B$B<QY[fhm  C  VZ$B  %[  %[!RXa[]***!#A!7!' Y.99;;BB$9$D$D$F$F#WXXBB	r#   c                 "D  xyz |j         r|S t          j                    }d|| j        d}|zdzz  }	|dv }
|dv }|dv r
d}d|d	<   d }nwd
}d|d	<   |d                                         dk    }|j        |                                         t          j                            d          z
  	                                }|
                    d| d|ddd|z  dd|           g d}|
r	|dk    rd}nd}dzcxk     rdk     rn n|
                    d|           |S |                                                    d          }|j        j         |j                            |          |_        n/|j        j        |k    r|j                            |          |_        |j        d         }|d         dk    }|||                                                             d           z  }|                                o#t)          j        |          d         d         dk    }| }t)          j        |t)          j        |d          z            d         }t/          |          dk    rt)          j        |          d         }t/          |          dk    rd }nt1          |d                   }d!| d"| }|&|d#|j        |         	                                 dz  }|
                    ||           |                                }|                    g d$dd%&          }|                    dd'i(          }|d'                                         }|                                rzd)|d*<   d+|j        |d*f<   |d'         |          d,z                      d-                              d.          d/z   |j        | d*f<   |d*         |d'<   |                    d*d            n7|d'         d,z                      d-                              d.          d/z   |d'<   d0gy|                    yfd1|D             dd%&          }|
rF|dk    r@zd2vr<t)          j        |d3fd          }d4d0g}||                                         }|dk    }n9t)          j        |d5fd          }|}||                                         }|dk    }|j        j        s|                                }|                                rd||<   |d0         dk    }|                                rit)          j         t/          |                    }|d6                                         |          |d0                                         |          z  || <   n5|d6                                         |d0                                         z  }|j!        }|t(          j"        k    r|                    d7          }tG          |j        d                   D ];} |d d | fxx         |z  cc<   ||          |j$        v r|||          xx         |z  cc<   <|t(          j"        k    r|                    d-          }|dd f         |d d8f         z  |dd <   |t)          j        |dd           z  }!|!                                rd||!<   |
r|dk    rt)          j%        |d           }"nt)          j&        |d           }"t)          j'        |"          }#|#                                rd|"|#<   tQ          z|	          }$t)          j(        |"          |$dz
  d9z  dz   k     rAt)          j        |"          d|$dz
  d9z  dz   z  k    r|
                    d:| d;|           |S t)          j)        |"d<d=g          \  }%}&|&|%z
  }'|%d>|'z  z
  }(|&d>|'z  z   })|"|(k    |"|)k    z  }t)          j*        |"|                   }*t)          j+        |"|                   }+|+|*z  },|
                    d?|*d@dA|+dd#|,dBz  dCdD|           dE|,z  }-|
r|dk    r|-dFz  }-|dGv r|-d3z  }-tQ          z|	          d|-z   k     r8|
                    dH|           |
                    dI|,ddJ|-d|           |S |"|	z  }.tQ          z|	          }$|
                    dK|$dLdM|-d|           |$dz   |-z   d9z  }/|
                    dN|/dLdOd|/z  dL|           dP|j$        vrd|dP<   |rdt)          j        |d5fd          }||         ,                    dd                                          }|}|dd f         |d d8f         z  |dd <   n|"}|r^tG          t/          |                    D ]@} ||          }0|d d | f         ||0dQz   <   ||0dQz            -                    dF          ||0dQz   <   An#|"|dR<   |dR         -                    dF          |dR<   ||	z  }.|d|/z  k     }1||/k    }2t/          |2j                  }3|3dk    r|2n|2                    d           }4|4                                r|4 }5t)          j        |5          d         }6t/          |1j                  }7|j        |7dk    r|1n|1                    d                    }8t)          j        |4          d         D ][}9|9dz
  }:|j        |:         };|d         j.        |:         }<|<dk    rdn|d         j.        |:dz
           |<z  }=|
                    dS|=d           |rV|:dz   t/          |          k     r@|d         j.        |:dz            }>|>dk    r$tQ          |=|d         j.        |:         |>z            }=|5d |:         /                                }?tQ          d|?dTz
            }@t          t/          |6          |@dUz   dz             }A|j.        |6|@|A                  }B|B                                }B|8|8|;k             }Ct/          |C          dk    rRt          |C          }D|D|;k    rd }En^|Bj        |;ta          j1        d          z   |Dta          j1        d          z
           }En$|Bj        |;ta          j1        d          z   d          }E|E	|Ej         rd }EdV }FdW }G|E~ |G|<|E          }H |G|Ed         j.        d         |E          }ItQ          |H|I          d3k    rA|
                    dX|;	                                 dY           |3dk    rd|2|9<   Rd|2|9d d f<   ]|1|2z  }|sF|.|dZ<   |1|d[<   |2|d\<   |dZ         -                    d3                              d.          |dZ<   ||d]<   ntG          t/          |                    D ]} ||          }0|.d d | f         ||0d^z   <   ||0d^z            -                    d3                              d.          ||0d^z   <   |1d d | f         ||0d_z   <   |2d d | f         ||0d`z   <   |d d | f         ||0daz   <   t)          j        |          d         D ]}9|j        |9         };t          t/          |          dz
  |9d3z             }J|2                    db          rdc}Kn|2                    d/          rdd}KndF}KtQ          d|9|Kz
            }L|j.        |L|J         }M|r|}Nndeg}N|ND ]r}0|0dek    r(|MdR         |Md]                                                   }On-|M|0dQz            |M|0daz                                                      }Ot)          j*        |O          }*t)          j+        |O          }+|+|*z  },dE|,z  }-|
r|dk    r|-dFz  }-|dGv r|-d3z  }-|$dz   |-z   d9z  }/|r||0dQz            j.        |9         }P|dR         j.        |9         }P|P|/k     r|Pd|/z  k    r|rF|
                    df|0 dg|;	                                 dh|,ddz  dCdi           d|j        |;|0daz   f<   4|
                    dj|;	                                 dh|,ddz  dCdi           d|j        |;d]f<   t|s;|1|d]                                         z  }1|2|d]                                         z  }2nitG          t/          |                    D ]L} ||          }0|0yv r>|1d d | f         ||0daz            z  |1d d | f<   |2d d | f         ||0daz            z  |2d d | f<   M|1|2z  }|                                s|
                    d:| d;|           |S dU}Q|d                                         dk    }R|dv r|R                                rt)          j        |R          d         }St)          j        |          d         }Tt/          |S          rt/          |T          sd S |Td d d f         |Sz
  }U|Ud8z  }U|Udk    }V|V                                r#|U|V                                         }Wt          j3        |          |Wz  }Xti          |Xt          j        j                  r!t          j                            |Qk          }/nta          j1        |Qk          }/ti          |/t          j        j                  r]ti          |Xt          j        j                  r>t)          j        |U|Wk              d         d         }9|j        |9         };|;|Xz   |;|/z   k     }Yn|X|/k     }Y|Yr|5                    dl|           |S |6                    tn          j8                  r^ts          tG          d|j        d                             |dm<   |j        d         dz
  |dm         z
  |dn<   |rN|do yD                                          d           |dp yD                                          d           z  }Zn|d[         |d\         z  }Z|Zt)          j        |Zd8          z  t)          j        |Zd          z  t)          j        |Zdq          z  t)          j        |Zd3          z  }Ztu          j;        drd dsdcdtdu          5  |
                    dvty          ||Z                   z              d d d            n# 1 swxY w Y   zfdw}[|0|j        d         dz
  |z
  }\|
                    dx| dy|\ |           |rYt)          j        |d          }]|r*t)          j        |d          }^t)          j        |d          }_g dzxtG          t/          |                    D ]} ||          }0t)          j        |          d         d         }`|r|||`k    r|d d | f         }a|2d d | f         }b|1d d | f         }c |[|a|d          |b|d          |c|d                    }dt/          |d          dk    rGtG          t/          |d                    D ]*}:|d|:         }.|.d         |z   |.d         |z   |.d3         f|d|:<   +t)          j=        t)          j        |bd8                    }et)          j=        t)          j        |cd8                    }f|f|ez  }g |[|g|\d          |f|\d          |e|\d                    }ht/          |h          dk    rtG          t/          |h                    D ]*}:|h|:         }.|.d         |\z   |.d         |\z   |.d3         f|h|:<   +tG          t/          |h                    D ]*}:|h|:         }.||.d         z
  ||.d         z
  |.d3         f|h|:<   +|d}i|i>                    |h           n+ |[|d d | f         |2d d | f         |1d d | f                   }i|
                    d{|0 d||i |           |tG          t/          |i          dz
  d8d8          D ]|}:|i|:         }.|j        |.d                  	                                |k     rI|
                    d}|0 d~|j        |.d                   d|j        |.d         dz
            d|           |i|:= }t/          |i          dk    r|ix| <   t_          d xD                       }j|jdk    rn|jdk    rxfdtG          t/          |                    D             }kt)          j        t)          j?        |k          d8k              d         d         }9||9         }l|
                    d| d|l d|           nfg d}mtG          t/          |                    D ]} ||          }0x|          }i|ig }i|iD ]}.|.d3         dk    rz}n|	}on|	}nz}o|
r\d| d|0 d|j        |.d         dz
           	                                 d|j        |.d                  	                                 d|nd
}pn7d| d|0 d|j        |.d         dz
            d|j        |.d                   d|nd
}p|
                    |p|           |m| xx         |.d         |.d         z
  z  cc<   |j.        |.d         |.d         |j$        @                    |0          fxx         |nz  cc<   |0d0k    r=|j.        |.d         |.d         |j$        @                    d6          fxx         |nz  cc<   |r3|0d4k    rd|^|.d         |.d         <   n|0d0k    rd|_|.d         |.d         <   d|]|.d         |.d         <   t_          |m          dk    rkd)}qtG          t/          |                    D ]0} |m|          dk    r"|qd)k    r|qdz  }q|q||           d|m|           dz  }q1d|q }p|5                    |p|           |r|^|_z  }rt)          jA        |^|_          }s|r                                rA|j        |rdf         |oz  -                                                    d-          |j        |rdf<   |s                                rD|j        |sdf         d9z  |oz  -                                                    d-          |j        |sdf<   d|j        |]dPf<   njd}mt)          j        |          d         d         }`|r|||`k    r |[||d          |2|d          |1|d                    }dt/          |d          dk    rGtG          t/          |d                    D ]*}:|d|:         }.|.d         |z   |.d         |z   |.d3         f|d|:<   +t)          j=        t)          j        |2d8                    }tt)          j=        t)          j        |1d8                    }u|u|tz  }v |[|v|\d          |u|\d          |t|\d                    }ht/          |h          dk    rtG          t/          |h                    D ]*}:|h|:         }.|.d         |\z   |.d         |\z   |.d3         f|h|:<   +tG          t/          |h                    D ]*}:|h|:         }.||.d         z
  ||.d         z
  |.d3         f|h|:<   +|d}i|i>                    |h           n |[||2|1          }i|tG          t/          |i          dz
  d8d8          D ]y}:|i|:         }.|j        |.d                  	                                |k     rF|
                    d|j        |.d                   d|j        |.d         dz
            d|           |i|:= z|iD ]Q}.|.d3         dk    rz}n|	}on|	}nz}o|
                    d|. d|n |           dD ]?}0|j.        |.d         |.d         |j$        @                    |0          fxx         |nz  cc<   @|r=|j.        |.d         |.d         |j$        @                    d          fxx         |nz  cc<   |rw|j$        @                    d          }w|j.        |.d         |.d         |wf         |oz  -                                                    d-          |j.        |.d         |.d         |wf<   d|j.        |.d         |.d         |j$        @                    dP          f<   |.d         |.d         dz
  k    rH|
r,d| d|j        |.d                  	                                 }pnd| d|j        |.d                   }pnh|j        |.d         dz
           }@|j        |.d                  }A|
r1d| d|@	                                 d|A	                                 d}pnd| d|@ d|A d}p|
                    |p|           |m|.d         |.d         z
  z  }mSt/          |i          d3k    r\d}p|iD ]V}.|pd|j        |.d         dz
           	                                 d|j        |.d                  	                                 z  }pWnd|m d}p|5                    |p|           |r|d                                         }#|#                                rA|d         |#          -                    d                              d-          |j        |# df<   n1|d         -                    d                              d-          |d<   |                                S )Nzprice-change-repairrJ  r9  r&  r'  )r  ru  z
100x errorr  rK  z	bad splitzprice-repair-splitrm   rw   r/   r[  z
start_min=z change=rS  z (rcp=rV   rX  )rd   re   rf   rg   r$   TFr  g      ?z(Split ratio too close to 1. Won't repairr  r   ri   ro   zappears_suspended=z, idx_latest_active=rS   )rh   rq   rm   r~   r   )rp   r   Volr   rR   VolStrNaNg    .Ar   r   rP   rg   c                     g | ]}|v|	S r   r   )r   r   
debug_colss     r!   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>
  s#    'O'O'Oa1J;N;N;N;N;Nr#   )r  rv  gMbP?rZ  rd   r2   rh   r  rN   r2  zNo z
s detected   K   r  z+Estimation of true 1D change stats: mean = z.2fz, StdDev = r  rT  z
% of mean)r(   r  )r*   r-   z>Split ratio too close to normal price volatility. Won't repairz	sd_pct = z  largest_change_pct = z
split_max=z.3fz largest_change_pct=z
threshold=z, threshold_rcp=r~   z 1D %z1D %z- vol_change_pct =    rF   c                    t          j        |j        |z
                                            }||                                z  }t          j        |           }t          j        |          t          j        |          z  }|d                                         }t          j        ||z            }t          j        |||z
  dz  z            }t          j	        |          }| |z
  |z  }	|	S )Nri   rZ  )
r   r  rj   r  r5   expr  r   r|  sqrt)
rs  r  block	distancesrt  rY  weighted_meanweighted_varianceweighted_stdz_scores
             r!   _calc_volume_zscore_weightedzLPriceHistory._fix_prices_sudden_change.<locals>._calc_volume_zscore_weighted^  s     "b(8'G'G'I'I J JI0I fiZ00G hw//"&//AG"8_5577F$&F6G+;$<$<M(*w&=:PUV9V/V(W(W%#%7+<#=#=L  &5EG"Nr#   c                     |d                                          }t          j        |d          }|dk    rdS t          j        |          }| |z
  |z  }|S )Nri   r/   )ddofrw   r   )r|  r   stdrW  )rs  r>  rY  rG  rW  rC  s         r!   _calc_volume_zscorezCPriceHistory._fix_prices_sudden_change.<locals>._calc_volume_zscorem  sY    "8_5577F&a000Cczz q76??D%}3G"Nr#   z'Detected false-positive split error on z, ignoring price droprdownupr  _r_down_up_frN  r%   rv  zn/az	Unusual 'z' price action @ zM is actually similar to local price volatility, so ignoring (StdDev % mean = z%)zUnusual price action @ r3   z<100x changes are too soon after stock split events, abortingrS  i_revc                     g | ]}|d z   S )rM  r   r  s     r!   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>  s    'F'F'Fa'	'F'F'Fr#   c                     g | ]}|d z   S )rN  r   r  s     r!   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>  s     bbbopcdejcjbbbr#   zdisplay.max_rowszdisplay.max_columnszdisplay.widthr-  z!price-repair-split: my workings:
c                    | d         rKt          j        |           } d| d<   t          j        |          }d|d<   t          j        |          }d|d<   |                                 sg S t          j        |           d         }g }t	          t          |          dz
            D ]]}|dz  dk    rRdk    r|||                  rdnd}n|||                  rdnd}|                    ||         ||dz            |f           ^t          |          dz  dk    rVdk    r||d                  rdnd}n||d                  rdnd}|                    |d         t          |           |f           |S )	Nr   Fr/   rZ  r9  r   z	1.0/splitrN   )r   r~  r   r  r@   r   r  )r  f_upf_downtrue_indicesrangesrS  r  r   s          r!   map_signals_to_rangeszEPriceHistory._fix_prices_sudden_change.<locals>.map_signals_to_ranges  s   t "GAJJ!wt}}Q!q	5577 	8A;;q>LF3|,,q011 O Oq5A::s{{)/Q)@Qggk-3LO-DQkk'MM<?LQ4G"MNNN<  1$))3;;%+L,<%=N'';CC)/R0@)AN++wC|B/Q=>>>Mr#   zidx_latest_active=z, idx_rev_latest_active=)NNNNzcolumn 'z
' ranges: zPruning z range r`  z because too old.c                     g | ]}|dnd	S )Nr/   r   r   )r   r  s     r!   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>\  s     PPPqamPPPr#   c                 (    g | ]}|         r|nd S rN   r   )r   rS  OHLC_correct_rangess     r!   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>a  s(    VVV03;VVVr#   z
Potential z detected only in column z(, so treating as false positive (ignore))r   r   r   r   r   z
Corrected z on col=z range=[:z] m=r]   =r  zCorrected: zPruning range zrange=z m=)rd   re   rf   rg   rh   rq   z on interval z across intervals rU   z (inclusive)z
Corrected:rT   )Br   r   r   r   r|  rj   r   r  r  r   r   r~  r   r   r   r   r   r   r   r   r   r  rollr   r   r   r,  r   r   fullr  r  onesr  r   r@   r   r  r  r  r5   
percentilerW  rG  r  r   r   r   r   r   r5  r   r   r{  isEnabledForr  DEBUGr   r   option_contextr   flipextendr  r  logical_xor){r   r   rA   r  changer(  r.  r   r  	split_rcpr*  multidayfix_type	start_minr  OHLCcorrect_columns_individuallyrH  r  f_no_activityappears_suspendedf_activeidx_latest_activelog_msgdf_workingsfna_1d_change_xprice_data_cols
price_dataf_zeror  df_dtyper  f_zero_num_denom_1d_change_denoisedf_na	split_maxq1q3iqrlower_boundupper_boundavgsdsd_pctlargest_change_pctrI  r  r   rV  rU  
f_up_ndimsf_up_shiftsnf_up_shiftsflat_indicesf_down_ndimsdown_dtsr  rS  r  r  vol_change_pctnext_vi_pos_in_flat_indicesr   r   r>  down_dts_fromnext_down_dtblock_afterrD  rH  z_score_afterz_score_after_d1idx_endlookback	idx_startchanges_localr.  clean_changes
big_changethreshold_daysf_splits	indices_A	indices_Bgapsf_posgap_mingap_tdwithin_thresholdr?  rY  idx_rev_latest_activef_correctedf_open_fixedf_close_fixedidx_first_fr*  f_upjf_downjranges_beforef_rev_downj	f_rev_upjf_revjranges_afterrX  r4  idxscoln_correctedrP   m_rcpr  counts_prettyf_open_and_closed_fixedf_open_xor_closed_fixed
f_rev_downf_rev_upf_revcol_locr]  r7  r   s{                                                                                                                           @@@r!   r/  z&PriceHistory._fix_prices_sudden_changeK
  s(   8 	I$&& 5h]a]hii
%K	::44]""#H#6Jx II"H#7Jx >"++--4A!**Y-D-R-RYZ-R-[-[[aaccIX)XXVXXXs6zXXXX`jkkk/// 	1D(( ,0((+0( LLC:LVVVIggii""U"339<	--k::CIIY\[((	,,[99CIIaL Hq(%D	(8(8(<(<!(<(D(DD)--//TBH]4K4KA4Nq4QST4T!>HX!0D0D%DEEaH !!Q&& " 2 21 5 !!Q&& $ #$5a$8 9 9a'8aaN_aa(BCI&78==??BBBBGWJ///hhjj!&&'^'^'^efow&xx!(((E1B(CC% %%''7799 	\$&K!-2KOCM*/:5/A3$/G/K.S.STY.Z.Z.a.abg.h.hkn.nKOSD(N+!,X!6K9I9I(YZ9I9[9[9[9["-e"4S"8!@!@!G!G!N!Nu!U!UX[![KY
!&&'O'O'O'O4'O'O'OVW`h&ii  	'D((U:M-M-M 7Aq63//L%g.O_-6688J3&FF7Aq63//L"O_-6688J3&F) 	+#**J::<< 	%!$Jv W"::<< 	H'#c((##C{+4466w?#g,BWBWBYBY[aZaBbbCLLk"++--G0E0E0G0GGC#rx#**733Jz'*++ 	7 	7AqqqsOOOs"OOOAw+---OA.///36///rx#**511J%abbc+j".??QRR!BGFAA$>$>$>>!! 	1-0L)* 		BD(("$*\"B"B"B #%)Lq"A"A"Ax+,,88:: 	,(+% y))	6%&&)a-3)>)BBBrvNaGbGbehmvyzmz  B  mB  EF  mF  fG  HG  HGLL3x333:LFFFI 2RH==B2g39n39n K/4G;4VWg)!,--V'*++c}3}}}WY}}}aghmam}}}}  FP  	Q  	Q  	Q Z 	(D((!#>))"a'"ui  3);#;;;LLYakLlllLL`V```HZ```hrLsssI  )+y))	])]]]EW]]]eoppp_'99S@	T)TTTYTTT\fgggck))$C' 	/7Aq63//LT**344==??J"O  *!""#/*SbST2CCL.L' 	?3//00 I I#A&)5aaad);AgI&)4QwY)?)E)Ea)H)HAgI&&I
 #6K"-f"5";";A">">K9$i/ i'__
(!mmddq1A1A?? f	0'<L8L11!4Lv|,,Ly<??

PQ
@R@RSHx,,Q/ a0 a0EYq\M&q)&'1ff#h-2DQqS2IA2MG>GGGHHH ]1s3xx ]/!4Fzz),^S]=OPQ=RU[=[)\)\2 )5RaR(8(<(<(>(>%A4r9::#l++U2XaZ88eCi!89((** (2 6}%%))#&}#5#5L#r)) '+&+i93Fq3I3I0I,W`WjklWmWmJm0m&n"')By/B1/E/E,E,F,F"GK*{/@*"&K# # ## # # *%8%8K%H%HM':':;x;P;UVW;XZe'f'f$
 =*:;;a?? %orwwyy%o%o%oppp%??(-DII*/DQQQKTM+ 	. K"(K $K*3/55a88??FFK K3//00 . .#A&&'1gAdF#&1!D&&9&?&?&B&B&I&I%&P&PAdF#)/1AgI&'+AAAqDzAeG$&'1gAdF## 8A;;q> )	= )	=C3B#c((1*c!e,,G  %% ""3'' As8|,,I',Yw->?M+ &w = =::$1&$9=;M:M$N$W$W$Y$YMM$1!G)$<mAdF>S=S$T$]$]$_$_Mgm,,VM**c%&Z" 0D 0 0&!+&>11*a/*&_/AASH	/ =!,QwY!7!<S!AJJ!,V!4!9#!>J!I--*s9}2L2L7 ="LL  *tQ  *t  *t  *t  *t  bh  il  bl  *t  *t  *t  *t  u  u  u:?KOB$J77"LL  *n27799  *n  *n  \b  cf  \f  *n  *n  *n  *n  o  o  o7<KOBG45=6 , 	Bk#.77999F+c*33555DD3//00 B B#A&
??#)!!!Q$<+af2E#EF111a4L!%aaadk!D&.A!ADAJTMuuww 	LL3x333:LFFFI ~&//11S8]""x||~~"**1-IAIy>> Y tQQQW%	1DBJD1HEyy{{ u+//++5h??'Ifi&=&KLL I ) 7 E E> E Z ZII ) 3 H H HIi)@)NOO :T^_egpg~  hM  UN  UN :(4=11!4Q7C3B(*VY'G$$'-	'9$# KK ^fpKqqqIw}-- 		e#E![->q-A$B$BCCK#.#4Q#7#9K<L#LK + C&'F'F:'F'F'FGKKQRKSSVabbt~bbb  WA  WE  WE  KL  WE  WM  WM  M&v.T1BB"'(B"7"77"'(A:N:NNQSQXYaceQfQffikipqy{|i}i}}H"#5t=RTVXgimnn e eF[YaMbIcIccddde e e e e e e e e e e e e e e	 	 	 	 	B ($&HQK!O6G$G!LLp.?ppYnpp  yCL  D  D  D' E	/'!U++K 2  "wq%00 "5 1 1":":":3t99%% -4 -4G hqkk!nQ/$ V*;*GL]alLlLl 111a4B AJE$QQQTlG$9$9"=N=O=O:PRWXiXjXjRkmt  vG  vH  vH  nI  %J  %JM=))A--!&s='9'9!:!: j jA -a 0A01!7H0H!A$QbJbdefgdh/iM!,,"$'"'%*<*<"="=K "(<(< = =I&4F#8#8@U@V@V9WYbcxcycyYz  }H  I^  I_  I_  }`  $a  $aL<((1,,!&s<'8'8!9!9 q qA ,QA/0t6K/KQqTTiMiklmnko.pLOO!&s<'8'8!9!9 E EA ,QA/01vq1vqt.DLOO*FMM,////221QQQT7DAJqqqRStUUF===V==ZPPP("3v;;q="b99 * *"1I9QqT?//11I=="LL)uA)u)uci!o)u)uQTQZ[\]^[_`a[aQb)u)u)u  ~HL  I  I  I &q	v;;??-3'*PP<OPPPQQEzz!VVVVU3t99EUEUVVVhrx~~344Q7:3iz(zzSzzz  CM  N  N  N  N (iis4yy)) 6 6AQA03F~!## 6 6Q47?? %A$-EE )A$)E# } #Kx  #K  #K  #K  #KCIVWXYVZ[\V\L]LbLbLdLd  #K  #Kgjgpqrstqugvg{g{g}g}  #K  #K  DE  #K  #K  #KCC"|x"|"|"|"|CIVWXYVZ[\V\L]"|"|`c`ijklmjn`o"|"|uv"|"|"|CS
;;;#A!A$qt)31adCK,?,?,B,B!BCCCqHCCC<<HQqT!A$Y0C0CK0P0P%PQQQUVVQQQ) @ F{{:>QqT!A$Y 7 7!"g;?ad1Q4i 815AaD1I..-6. {##a''$&M"3t99-- L L&q>Q..,22 - 5)Q-K-K+a.-K-K-KKM777CKK:K666 R*6*F'*,.}*U*U'*..00 LBE'JackJkBlotBtA{A{A}A}  BE  BE  FK  BL  BLCG3X=>*..00 RBE'JackJkBlorBruzBz  BB  BB  BD  BD  BK  BK  LQ  BR  BRCG3X=>04CGK,-- K(1++a.+K  @&7&CHY]hHhHh !6 5a8I8J8J6KTRcRdRdMegmn  oA  oA  hB  !C  !C}%%))"3}#5#566 f f)!,,-aD3D,DadM^F^`abc`d+ea((WRWT2%6%677
72762#6#677 :-44U;P;Q;Q5RT\]r]s]sTt  wA  BW  BX  BX  wY   Z   Z|$$q(("3|#4#455 m m(O+,Q42G+G1PeIeghijgk*lQ"3|#4#455 A A(O+,QqT61QqT61Q4*@Q&l++++..q$??$s6{{1}b"55 & &Aq	Ay1++--	99%mci!o%m%mSTUVSWXYSYIZ%m%m%mu  A  A  A"1I + +Q47??A%EE!A!E/a//A//zBBBF E EAHQqT!A$Y(;(;A(>(>>???1D????# OHQqT!A$Y(;(;K(H(HHIIIQNIII! p!k11(;;G47HQqT!A$Y=O4PSX4X3_3_3a3a3h3hin3o3oCHQqT!A$Y/0HL1adCK$7$7$D$DDEQ41Q4!8## TZ8ZZ#)AaD/BVBVBXBXZZS8SS#)AaD/SS  IadQh/E)AaD/C dq8qquzz||qqY\YaYaYcYcqqqc8ccuccRUcccS
333qtad{*6{{a" V VAUsy1a05577UUSYqt_=Q=Q=S=SUUUCCV 3K222KK:K... 	Ex=%%''Dxxzz E*-h-*>*D*DQ*G*G*N*Nu*U*Uh'' #H 3 3A 6 6 = =e D DH~~s   P,AQQAQQAQr  )r5   r$   F)r5   F)r5   r\  )F)FF)__name__
__module____qualname__r"   r   log_indent_decoratorr
   r   r   r  r  r   r  r   r#  r%  r'  r0  r   r  r   r   r   r  r  r   r   r   r   r/  r   r#   r!   r   r      s       0 0 0 0 +deT eE%"	y y (*|	y y y  yv. .r| . . . .	&d 	& 	& 	& 	&a a29 a a a ae ery e e e e^ ^	 ^ ^ ^ ^ 29    0. .", . . . .` A A A  AF7 7 7r% % % %N     X X  Xt c c  c  @ @  @D o o  ob ~ ~  ~@  : :  :x b
  b
  b
   b
  b
  b
 r#   r   )!	curl_cffir   mathr   r  r?  r   dateutilr  r  numpyr   pandasr   r   r   r   yfinancer   r   yfinance.configr   yfinance.constr   r	   r
   yfinance.exceptionsr   r   r   r   r   r   r   r#   r!   <module>r     s.                                      " " " " " " " " $ $ $ $ $ $ G G G G G G G G G G              ]3  ]3  ]3  ]3  ]3  ]3  ]3  ]3  ]3  ]3 r#   