a
    dF                     @   s  d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl	m
Z
 ddlZddlZddlZddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZ ddlm  mZ ddlZdd	lm Z m!Z!m"Z" ddl#m$Z% dd
l&m'Z' ddl(m  m)Z* ddl+m,Z,m-Z- ej.dd Z/dd Z0ej12ddde3de4de5dedg de4d6dde4d6ddddd e7dj8ddde7dj8ddde7dj8ddddddddf gdd Z9dd  Z:d!d" Z;d#d$ Z<ej12d%ej=e;d&d'ej=ej>d(d'ej=ejej>ej?d)d*d'ej=ejej>d+d)d,d'ej=ejej>d-d)d.d'gej12d/ej>e;gd0d1 Z@d2d3 ZAd4d5 ZBej.d6d7 ZCG d8d9 d9ZDG d:d; d;ZEej12d<d=gd>e g d?fd@ejFGg d?g dAgfgdBdC ZHdDdE ZIej12dFg dGdHdI ZJdJdK ZKG dLdM dMejLZMdNdO ZNdPdQ ZOdRdS ZPdTdU ZQej12dVdWdXdVedYdZdZd[gd\d] ZRd^d_ ZSd`da ZTejUdbdc ZVej12ddejWej?gdedf ZXdgdh ZYdS )ia<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )arrayN)partial)Path)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)IndexSeriesperiod_rangecreate_pickle_data)DayMonthEndc                   C   s   t  S Nr    r   r   d/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pandas/tests/io/test_pickle.pycurrent_pickle_data7   s    r   c                 C   s   t |trt||  d S |dr2t| | nP|dkrb|tju rT| tju s`J q| |ksJ n ttd| dtj	}|| | d S )NZsp_	timestampZassert_Z_equal)

isinstancer	   tmassert_index_equal
startswithZassert_equalpdZNaTgetattrZassert_almost_equal)resultexpectedtypZ
comparatorr   r   r   compare_element@   s    


r   datas   123s   123456I)         B)r#   r"   r"      )r#      C)orderFc                 C   s   t | }t| d}||ks"J t| ttfr>|| u sJ nBt|tr|jdksVJ |jdksdJ |jsnJ |j	|j
fksJ d S )NAr!   r$   )r   
memoryviewtobytesr   bytes	bytearrayndimformat
contiguousshapenbytes)r   r   r   r   r   r   test_flatten_bufferV   s    

r4   c           
   	   C   s  t  std ttjdD ]}| |}t|}|	 D ]\}}|	 D ]\}}|| | }|dkrR|dkrRt
|| |jj|jjksJ |jjjrJ t
|dk|dk |jj}|td tdksJ |tjdd }	t|	tjsJ |	tjddd	ksJ |tjdd
 }	t|	tjs8J |	tjdddksJ qR|dkr|dkrt
|| t|jtsJ |jt ksJ |jdksJ t
|d|d qR|dkr|dv rt
|| qR|dkr|dv rt
|| qRt||| qRq@q d S )Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*Zseriestsr   r!   r"   )hours)daysr6   )nanoseconds)r7   r8   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   __file__parentglobr   read_pickleitemsr   assert_series_equalr9   freq	normalizer   Z	Timedeltar   r   r   Zfreqstrshiftassert_frame_equalr   )
datapathlegacy_pickler   r   dvdtr   r   rE   resr   r   r   test_pickless   s@    

rN   c                 C   s>   t |d }tj| |dd W d    n1 s00    Y  d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler   s    r\   c                 C   sB   t | d$}|d t|W  d    S 1 s40    Y  d S )Nrbr   )rU   seekrV   load)rZ   r[   r   r   r   python_unpickler   s    
r`   pickle_writerpython)idZpandas_proto_defaultrR   Zpandas_proto_highestr&   Zpandas_proto_4   Zpandas_proto_5writerc                 C   s  | }|  D ]\}}|  D ]\}}t }||| t|}	t|	|| t|}	t|	|| t|dd$}
||| |
d W d    n1 s0    Y  t|dd$}
t|
}	|
d W d    n1 s0    Y  t|	|| W d    q1 s0    Y  qqd S )NrP   moder   r]   )	rC   r   ensure_cleanr   rB   r   r`   rU   r^   )r   ra   re   r   r   rK   rL   r   rZ   r   handler   r   r   test_round_trip_current   s     



(
(rj   c                  C   s(   t  } t | jtj}t | | d S r   )r   makeDataFrameround_trip_pathlib	to_pickler   rB   rH   dfr   r   r   r   test_pickle_path_pathlib   s    rp   c                  C   s(   t  } t | jtj}t | | d S r   )r   rk   Zround_trip_localpathrm   r   rB   rH   rn   r   r   r   test_pickle_path_localpath   s    rq   c                   C   s   dt   dS )N__z	__.pickle)uuiduuid4r   r   r   r   get_random_path   s    ru   c                   @   sT   e Zd ZejZdd Zdd Zej	
dg ddd Zd	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C   s  |d u rt || d S |dkr0t|d}n4|dkrHt|d}n|dkrtj|dtjd$}|	|t
j| W d    n1 s0    Y  n|dkrt|d^}tj|dd0}||t
j|}||| W d    n1 s0    Y  W d    n1 s0    Y  nF|d	kr6t |d}n.|d
krRtd|d}nd| }t||dvrt|dD}| |	|  W d    n1 s0    Y  W d    n1 s0    Y  d S )Ngzipwbz2zipcompressiontarr]   rf   xzZzstdZ	zstandardrP   zUnrecognized compression type: )rz   r}   )shutilcopyfilerw   rU   ry   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosrZ   basenametarfile
gettarinfoaddfiler   r   
ValueErrorread)	selfZsrc_path	dest_pathr|   fr[   r}   tarinfomsgr   r   r   compress_file   s2    4
L



zTestCompression.compress_filec                 C   s  |}|d }|d }t |}t |}t  }|j||d t j||dH}	t|d}
|
|	  W d    n1 s0    Y  W d    n1 s0    Y  tj	|d d}t 
|| W d    n1 s0    Y  W d    n1 s0    Y  d S )N.compressed.rawr{   rP   )r   rh   rk   rm   decompress_filerU   r   r   r   rB   rH   )r   r|   ru   basepath1path2p1p2ro   r   r[   df2r   r   r   test_write_explicit  s    Jz#TestCompression.test_write_explicitr|   ) NonebadZ7zc              	   C   sr   t jtddP t|&}t }|j||d W d    n1 sF0    Y  W d    n1 sd0    Y  d S )NzUnrecognized compression type)matchr{   )r=   Zraisesr   r   rh   rk   rm   )r   r|   ru   rZ   ro   r   r   r   test_write_explicit_bad&  s    z'TestCompression.test_write_explicit_badc                 C   s  |}|| }|d }| j | }t|}t|}t }	|	| tj||dH}
t|d}|	|

  W d    n1 s0    Y  W d    n1 s0    Y  tj|d d}t|	| W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   r{   rP   )_extension_to_compressiongetlowerr   rh   rk   rm   r   rU   r   r   r   rB   rH   )r   compression_extru   r   r   r   r|   r   r   ro   r   r[   r   r   r   r   test_write_infer-  s    
Jz TestCompression.test_write_inferc           
   	   C   s   |}|d }|d }t |z}t |P}t  }|j|d d | j|||d tj||d}	t ||	 W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   r   r{   )r   rh   rk   rm   r   r   rB   rH   )
r   r|   ru   r   r   r   r   r   ro   r   r   r   r   test_read_explicitC  s    z"TestCompression.test_read_explicitc              	   C   s   |}|d }|| }| j | }t|v}t|L}t }	|	j|d d | j|||d t	|}
t
|	|
 W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   r{   )r   r   r   r   rh   rk   rm   r   r   rB   rH   )r   r   ru   r   r   r   r|   r   r   ro   r   r   r   r   test_read_inferU  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__icomZextension_to_compressionr   r   r   r=   markparametrizer   r   r   r   r   r   r   r   rv      s   
rv   c                   @   s&   e Zd Zejdg ddd ZdS )TestProtocolrS   )rQ   r   r!   r"   c                 C   sZ   t |<}t  }|j||d t|}t || W d    n1 sL0    Y  d S )NrR   )r   rh   rk   rm   r   rB   rH   )r   rS   ru   rZ   ro   r   r   r   r   	test_reado  s
    
zTestProtocol.test_readN)r   r   r   r=   r   r   r   r   r   r   r   r   n  s   r   pickle_fileexcolsztest_py27.pkl)abcztest_mi_py27.pkl)r*   r$   r'   c                 C   s*   | ddd|}t |}t|j| d S )Nior   rV   )r   rB   r   r   columns)rI   r   r   rZ   ro   r   r   r   test_unicode_decode_errorx  s    
r   c               	   C   s   t  } t  }t| d}|| W d    n1 s<0    Y  t| d}t|}W d    n1 sp0    Y  t || W d    n1 s0    Y  d S )NrP   r]   )r   rh   rk   rU   rm   r   rB   rH   )rZ   ro   r[   r   r   r   r   test_pickle_buffer_roundtrip  s    
((r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                    s   dd }G dd d t  R fdd}t  }|| | d| t|}t || W d    n1 sv0    Y  d S )Nc                 S   s>   t |d }tj| |dd W d    n1 s00    Y  d S rO   rT   rX   r   r   r   r\     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s:   e Zd ZddddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponseNreturnc                 S   s.   t |d| _d|v r ddi| _n
ddi| _d S )Nr]   rw   zContent-Encodingr   )rU   fileheaders)r   rZ   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r   r   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  S r   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  S r   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                     s    S r   r   )r   kwargsr   rZ   r   r   mock_urlopen_read  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rh   rk   setattrr   rB   rH   )Zmonkeypatchr   r\   r   ro   r   r   r   r   test_pickle_generalurl_read  s    


r   c                  C   sb   t d t < d} t }||  t| }t|| W d    n1 sT0    Y  d S )NZfsspeczmemory://mockfile)	r=   Zimportorskipr   rh   rk   rm   r   rB   rH   )r   ro   r   r   r   r   test_pickle_fsspec_roundtrip  s    



r   c                   @   s   e Zd ZddddZdS )MyTzNr   c                 C   s   d S r   r   r   r   r   r   r     s    zMyTz.__init__)r   r   r   r   r   r   r   r   r     s   r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts@J d S )N)Zdtyper   r!   )r
   objectr   r   round_trip_picklerD   r   )r   r   r   r   r   test_read_pickle_with_subclass  s    
r   c                 C   s   t  }t  *}|j|| d t| }W d   n1 s@0    Y  t }|j|| d |d |	 |ks| dv sJ t
j|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r{   Nr   )rw   rz   r}   )r   rk   rh   rm   r   
read_bytesr   BytesIOr^   getvaluer   rB   rH   )r|   ro   rZ   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    
*

r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r   )r   r   rH   )r<   	unpickledr   r   r   _test_roundtrip  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr<   r   r   r   r   +test_pickle_dataframe_with_multilevel_index  s    
r   c                  C   sH   t dddd} ttjdt| | }t|}|j	j
dksDJ d S )Nz1/1/2011z1/1/2012r:   )rE   r"   )r   r
   nprandomZdefault_rngZstandard_normallenr   r   r9   rE   )Zprngr5   Znew_tsr   r   r   "test_pickle_timeseries_periodindex  s    
r   namei	  g     H@i     )r!   r"   c                 C   s$   t t j| d}|j| ks J d S )N)r   )r   r   ZmakeTimeSeriesr   )r   r   r   r   r   test_pickle_preserve_name  s    r   c                 C   s   t | }t ||  d S r   r   r   rD   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimes  s    
r   c                 C   s   t | }t ||  d S r   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_strings!  s    
r   c                  C   sf   t tddjdg } t| }|jjd jdks:J |jjd j	dksPJ t
|dg |  d S )Nabccategoryr   r!   )r!   T)r
   listZastypeZilocr   r   Z_mgrblocksr/   r2   rD   )ZserrM   r   r   r    test_pickle_preserves_block_ndim&  s
    
r   rS   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )rS   r|   r{   )	r   	DataFrameranger   rl   r   rm   rB   rH   )rS   r|   ro   r   r   r   r   %test_pickle_big_dataframe_compression3  s    r   c                 C   sh   | t tjdddd}t|d}t|}W d    n1 s@0    Y  tjg g d}t	|| d S )Nr   rJ   z1.2.4zempty_frame_v1_2_4-GH#42345.pklr]   )r9   r   )
r   r?   r@   rU   rV   r_   r   r   r   rH   )rI   rZ   fdro   r   r   r   r   #test_pickle_frame_v124_unpickle_130>  s    (r   )Z__doc__r   ry   datetime	functoolsr   rw   r   r   pathlibr   rV   r   r   rs   r   numpyr   r=   Zpandas.compatr   r   Zpandas.compat._optionalr   Zpandas.compat.compressorsr   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r	   r
   r   Zpandas._testingZ_testingr   Z-pandas.tests.io.generate_legacy_storage_filesr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   Zfixturer   r   r   r   r.   r+   PickleBuffercastZarangeZreshaper4   rN   r\   r`   paramrm   HIGHEST_PROTOCOLrj   rp   rq   ru   rv   r   Z
MultiIndexZfrom_arraysr   r   r   r   tzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   DEFAULT_PROTOCOLr   r   r   r   r   r   <module>   s   
$
2
~



%
	



