o
    &j?l                     @   sX  d dl Z d dlZd dlZd dlZd dlZzd dlZd dlmZ dZW n e	y-   dZY nw zd dl
mZ d dlmZ d dlmZ d dlmZ dZW n e	yU   dZY nw d	Zd
d ZG dd deZe ZG dd deZede  G dd deZdd ZG dd deZG dd deZG dd deZdd Zdd Z dd  Z!d!d" Z"dS )#    N)	HdfsErrorTF)	BosClient)	exception)BceClientConfiguration)BceCredentialsi   c                 C   s    t  }||  t| S N)hashlibmd5updatebase64standard_b64encodedigest)bufferr	    r   R/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/visualdl/io/bfile.pycontent_md5*   s   
r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )FileFactoryc                 C   s
   i | _ d S r   )_register_factoriesselfr   r   r   __init__1   s   
zFileFactory.__init__c                 C   s   | j ||i d S r   )r   r
   )r   path
filesystemr   r   r   register_filesystem4   s   zFileFactory.register_filesystemc                 C   s   | dr'd| jvr'tstdz	tdt  W n$ tjj	y&   tdw | dr>d| jvr>t
s7tdtdt  d}|d	}|d
krO|d | }| j|d }|d u r`td| |S )Nhdfs://hdfsz#Please install module named "hdfs".z,Please initialize `~/.hdfscli.cfg` for HDFS.bos://Zbosz-Please install module named "bce-python-sdk". z://r   z&No recognized filesystem for prefix %s)
startswithr   HDFS_ENABLEDRuntimeErrordefault_file_factoryr   HDFileSystemr   utilr   BOS_ENABLEDBosFileSystemfindget
ValueError)r   r   prefixindexfsr   r   r   get_filesystem7   s@   


zFileFactory.get_filesystemN)__name__
__module____qualname__r   r   r,   r   r   r   r   r   0   s    r   c                   @   sx   e Zd Zdd Zedd Zedd Zedd Zd	d
 ZdddZ	dddZ
dd ZdddZdddZdd ZdS )LocalFileSystemc                 C   s   d S r   r   r   r   r   r   r   W      zLocalFileSystem.__init__c                 C   s   t j| S r   )osr   existsr   r   r   r   r3   Z   s   zLocalFileSystem.existsc                 C   s   t |  d S r   )r2   makedirsr4   r   r   r   r5   ^   s   zLocalFileSystem.makedirsc                 G   s   t jj| g|R  S r   )r2   r   joinr   pathsr   r   r   r6   b   s   zLocalFileSystem.joinc                 C   s   t j|S r   )r2   r   isfiler   filenamer   r   r   r9   f      zLocalFileSystem.isfileTc                 C   sD   |rdnd}t ||}| }W d    |S 1 sw   Y  |S )Nrbr)openread)r   r;   binary_modemodereaderdatar   r   r   	read_filei   s   

zLocalFileSystem.read_fileFNc                 C   s   |rdnd}|r
d nd}d }|d ur| dd }t|||d }|d ur)|| ||}	d| i}
|	|
fW  d    S 1 sBw   Y  d S )Nr=   r>   utf-8last_offset)rB   encoding)r'   r?   seekr@   tell)r   r;   rA   sizecontinue_fromrB   rH   offsetfprD   continue_from_tokenr   r   r   r@   o   s   

$zLocalFileSystem.readc                 C   sN   d|v rd nd}t |||d}|| W d    d S 1 s w   Y  d S )NbrF   )rH   )r?   write)r   r;   file_contentrB   rH   rN   r   r   r   _write|   s   "zLocalFileSystem._writec                 C   D   z|  |||r	dnd W d S  ty!   | tj| Y d S w NabarS   FileNotFoundErrorr5   r2   r   dirnamer   r;   rR   rA   r   r   r   append   
   zLocalFileSystem.appendc                 C   rT   rU   rX   r[   r   r   r   rQ      r]   zLocalFileSystem.writec                 C   s"   dt jkrt j|ddS t |S )NposixT)followlinks)r2   namewalk)r   dirr   r   r   ra      s   

zLocalFileSystem.walkT)FNNF)r-   r.   r/   r   staticmethodr3   r5   r6   r9   rE   r@   rS   r\   rQ   ra   r   r   r   r   r0   V   s    






r0   r   c                   @   sh   e Zd Zdd Zdd Zdd Zddd	Zd
d Zedd Z	dddZ
dddZdddZdd ZdS )r"   c                 C   s   t j d| _d S )Ndev)r   configZConfigZ
get_clientclir   r   r   r   r      s   zHDFileSystem.__init__c                 C   s$   | j j|dd  ddd u rdS dS )N   F)	hdfs_pathstrictT)rh   statusr   r   r   r   r   r3      s   zHDFileSystem.existsc                 C      t |S r   r3   r:   r   r   r   r9         zHDFileSystem.isfileTc                 C   sD   | j j|dd  d}| }W d    |S 1 sw   Y  |S Nri   rj   )rh   r@   )r   r;   rA   rC   rD   r   r   r   rE      s   

zHDFileSystem.read_filec                 C   s   | j j|dd  d d S rq   )rh   r5   rm   r   r   r   r5      s   zHDFileSystem.makedirsc                 G   $   t jj| g|R  }|dd |S N\/r2   r   r6   replacer   r8   resultr   r   r   r6         zHDFileSystem.joinFr   Nc           
      C   s   d}|d ur| dd}|rd nd}z1| jj|dd  ||d}| }d|t| i}	||	fW  d    W S 1 s<w   Y  W d S  tyN   tdw )Nr   rG   rF   ri   )rj   rM   rH   zNo more events to read on HDFS.)r'   rh   r@   lenr   EOFError)
r   r;   rA   rK   rL   rM   rH   rC   rD   rO   r   r   r   r@      s"   (zHDFileSystem.readc                 C      | j j|dd  |dd d S Nri   T)rj   rD   r\   rh   rQ   r[   r   r   r   r\         zHDFileSystem.appendc                 C   r~   r   r   r[   r   r   r   rQ      r   zHDFileSystem.writec                 C   s$   | j j|dd  d}dd |D S )Nri   rr   c                 s   s$    | ]\}}}d | ||gV  qdS )r   Nr   ).0rootdirsfilesr   r   r   	<genexpr>   s   " z$HDFileSystem.walk.<locals>.<genexpr>)rh   ra   )r   rb   Zwalksr   r   r   ra      s   zHDFileSystem.walkrc   Fr   Nrd   )r-   r.   r/   r   r3   r9   rE   r5   re   r6   r@   r\   rQ   ra   r   r   r   r   r"      s    




r"   c                 C   s:   | dd  } |  d}| d| }| |d d  }||fS )N   rv   r      )r*   )r   r*   bucket_name
object_keyr   r   r   get_object_info   s
   
r   c                   @   s:   e Zd ZdddZdd Zdd Zedd	 Zd
d ZdS )BosConfigClientbj.bcebos.comc                 C   s&   t t||||d| _t| j| _d S )NcredentialsZendpointZsecurity_tokenr   r   rg   r   
bos_clientr   Zbos_akZbos_skbos_stsbos_hostr   r   r   r      s   zBosConfigClient.__init__c                 C   8   t |\}}z
| j|| W dS  tjy   Y dS w NTFr   r   get_object_meta_datar   BceErrorr   r   r   r   r   r   r   r3         zBosConfigClient.existsc                 C   d   | ds	|d7 }| |rd S t|\}}| ds|d7 }d}| jj|||t|t|d d S Nrv       r   keyrD   r   content_lengthendswithr3   r   r   append_objectr   r|   r   r   r   r   	init_datar   r   r   r5         



zBosConfigClient.makedirsc                 G   rs   rt   rw   ry   r   r   r   r6      r{   zBosConfigClient.joinc                 C   sN   |  |s
| | t|\}}| ||}td|  | jj|||d dS )a  !
        Upload a local file to baidu bos filesystem. The path can de divided as bucket name and prefix directory.
        The file would be uploaded in bucket at path `join(prefix directory in path, filename)`
        @param self object
        @param path(str)  bos directory path to store file, which consists of bucket_name + prefix directory.
        @param filename(str) local file path to upload
        zUploading file `%s`)bucketr   	file_nameN)r3   r5   r   r6   printr   Zput_object_from_file)r   r   r;   r   r   r   r   r   upload_object_from_file   s   


z'BosConfigClient.upload_object_from_fileNr   )	r-   r.   r/   r   r3   r5   re   r6   r   r   r   r   r   r      s    

r   c                   @   s   e Zd Zd$ddZdd Z	d%ddZd	d
 Zdd Zd$ddZdd Z	dd Z
dd Zedd Zd&ddZdd Zd'ddZd(d d!Zd"d# ZdS ))r%   Tc                 C   sH   d| _ d| _i | _d| _d| _t | _|r"|   t| j	| _
d S d S )Nr   r   r   )max_contents_countmax_contents_timeZfile_length_map_file_contents_to_add_file_contents_counttime_start_append_timeget_bos_configr   rg   r   )r   Z
write_flagr   r   r   r     s   
zBosFileSystem.__init__c                 C   s   t d}|stdt d}|stdt d}|s!tdtt dd| _tt d	d| _t d
}tt||||d| _dS )zC
        Get Bos configuration from environment variables.
        ZBOS_HOSTz${BOS_HOST} is not found.ZBOS_AKz${BOS_AK} is not found.ZBOS_SKz${BOS_SK} is not found.ZBOS_CACHE_COUNTr   ZBOS_CACHE_TIMEZBOS_STSr   N)	r2   getenvKeyErrorintr   r   r   r   rg   )r   r   Zaccess_key_idZsecret_access_keyr   r   r   r   r     s"   



zBosFileSystem.get_bos_configr   c                 C   s&   t t||||d| _t| j| _dS )zS
        Set Bos configuration and get bos client according to parameters.
        r   Nr   r   r   r   r   set_bos_config2  s   zBosFileSystem.set_bos_configc                 C   s   ddl }ddl}ddlm} t|d}||d }W d   n1 s%w   Y  |d }|j|d }|d}|d	}	d
|kr\|	d}
|	d}|	d}| |
|| dS t	d
|	 dS )zK
        Get bos client by visualdl provided ak, sk, and sts token
        r   N)CONFIG_PATHr>   
server_urlz/sts/)urlcodemsg000000sts_aksts_sktokenz%Renew bos client error. Error msg: {})requestsjsonZvisualdl.utils.dirr   r?   loadpostr'   r   r   format)r   r   r   r   rN   r   r   resZerr_coder   r   r   Z	sts_tokenr   r   r   renew_bos_client_from_server=  s"   




z*BosFileSystem.renew_bos_client_from_serverc                 C   rn   r   ro   r:   r   r   r   r9   S  rp   zBosFileSystem.isfilec                 C   s   t |\}}| j||}|S r   )r   r   get_object_as_string)r   r;   binaryr   r   rz   r   r   r   rE   V  s   zBosFileSystem.read_filec                 C   r   r   r   r   r   r   r   r3   [  r   zBosFileSystem.existsc                 C      | j ||S r   )r   r   )r   r   r   r   r   r   get_metac     zBosFileSystem.get_metac                 C   r   r   r   r   r   r   r   r5   f  r   zBosFileSystem.makedirsc                 G   rs   rt   rw   ry   r   r   r   r6   v  r{   zBosFileSystem.joinFr   Nc                 C   sp   t |\}}d}|d ur|dd}t| ||jj}||k r.| jj||||d gd}	nd}	d|i}
|	|
fS )Nr   rG   r   )r   r   ranger   )r   r'   r   r   metadatar   r   r   )r   r;   rA   rK   rL   r   r   rM   lengthrD   rO   r   r   r   r@   |  s    
zBosFileSystem.readc                 C   s(   | j | jkst | j | jkrdS dS r   )r   r   r   r   r   r   r   r   r   ready_to_append  s   zBosFileSystem.ready_to_appendc                 C   s  |  j |7  _ |  jd7  _|s|  sd S | j }t|\}}| |sd}z| jj|||t|t|d W nI t	j
t	jfy } z9|dkrt|   z	| j|| W n! t	jys   | jj|||t|t|d Y W Y d }~d S w |W Y d }~nd }~ww t|}	z| ||jj}
| jj|||t||	|
d W n7 t	j
t	jfy } z'|dkr|   | ||jj}
| jj|||t||	|
d n|W Y d }~nd }~ww d| _ d| _t | _d S )Nr   r   r   visualdl-server)r   r   rD   r   r   rM   r   )r   r   r   r   r3   r   r   r   r|   r   BceServerErrorBceHttpClientErrorr   r   r   r   r   r   r   r   )r   r;   rR   rA   forcer   r   r   er   rM   r   r   r   r\     s   



zBosFileSystem.appendc              
   C   s   |r| j ||dd d S t|\}}z| jj|||t|t|d W d S  tjtjfyU } z!|dkrH| 	  | jj|||t|t|d n|W Y d }~d S d }~ww )NF)rA   )r   r   rD   r   r   r   )
r\   r   r   Z
put_objectr|   r   r   r   r   r   )r   r;   rR   rA   r\   r   r   r   r   r   r   rQ     s6   zBosFileSystem.writec                 C   sh   G dd d}t |\}}|dv rd }n|dr|n|d }| jj||d}dd |jD }|||S )Nc                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
z)BosFileSystem.walk.<locals>.WalkGeneratorc                 S   s&   d | _ d| _|| _| | d| _d S Nr   )contentsr   r   handle_contentscount)r   r   r   r   r   r   r     s
   

z2BosFileSystem.walk.<locals>.WalkGenerator.__init__c              	   S   s   i }|D ]8}z| d}|d| }||d d  }W n ty(   d}|}Y nw || v r7|| | q|g||< qg }| D ]\}}|td| j |g |g qCt|| _	|| _
d S )Nrv   r   r   .r   )rindexr(   keysr\   itemsr%   r6   r   r|   r   r   )r   r   Zcontents_mapitemr   r   valueZ	temp_walkr   r   r   r     s*   


z9BosFileSystem.walk.<locals>.WalkGenerator.handle_contentsc                 S      | S r   r   r   r   r   r   __iter__  r1   z2BosFileSystem.walk.<locals>.WalkGenerator.__iter__c                 S   s.   | j | jk r|  j d7  _ | j| j d  S t)Nr   )r   r   r   StopIterationr   r   r   r   __next__  s   z2BosFileSystem.walk.<locals>.WalkGenerator.__next__N)r-   r.   r/   r   r   r   r   r   r   r   r   WalkGenerator  s
    r   )r   z./rv   )r)   c                 S   s   g | ]}|j qS r   )r   )r   contentr   r   r   
<listcomp>%  s    z&BosFileSystem.walk.<locals>.<listcomp>)r   r   r   Zlist_objectsr   )r   rb   r   r   r   r)   responser   r   r   r   ra     s   )
zBosFileSystem.walkrc   r   r   )FF)FT)r-   r.   r/   r   r   r   r   r9   rE   r3   r   r5   re   r6   r@   r   r\   rQ   ra   r   r   r   r   r%     s$    






Er%   c                   @   sx   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdddZ	dddZ
dd Zdd Zdd Zdd Zdd ZdS )BFilec                 C   sx   |dvrt d||| _t|| _t| jd| _d | _t	| _
d| _d | _d | _d| _d|v | _d|v | _d| _d S )N)r>   r=   brwwbbwzmode {} not supported by BFile.r\   r   FrP   r   )NotImplementedErrorr   	_filenamer!   r,   r+   hasattrfs_supports_appendbuff_DEFAULT_BLOCK_SIZEbuff_chunk_sizebuff_offsetcontinuation_token
write_tempwrite_startedrA   
write_modeclosed)r   r;   rB   r   r   r   r   *  s    


zBFile.__init__c                 C   r   r   r   r   r   r   r   	__enter__;  r1   zBFile.__enter__c                 C   s   |    d | _d| _d | _d S r   )closer   r   r   )r   exc_typeexc_valexc_tbr   r   r   __exit__>  s   
zBFile.__exit__c                 C   r   r   r   r   r   r   r   r   D  r1   zBFile.__iter__c                 C   s   | j |S r   )r+   r9   r:   r   r   r   r9   G  r<   zBFile.isfilec                 C   s:   | j }tt| j|| }|  j |7  _ | j|||  S )af  Read buffer from index self.buffer_offset to index new_buff_offset.

        self.buff_offset marks the last position of the last read,
        new_buff_offset indicates the last position of this read.
        self.buff_offset will be updated by new_buff_offset after this read.

        Returns:
            self.buff[i1: i2]: Content of self.buff.
        )r   minr|   r   )r   Znew_buff_offsetZold_buff_offset	read_sizer   r   r   _read_buffer_to_offsetJ  s   
zBFile._read_buffer_to_offsetTc                 C   r   r   )r+   rE   )r   r;   Zbinnaryr   r   r   rE   Y  r   zBFile.read_fileNc                 C   s   d}| j r2t| j | jkr2|dur*| | j| }t||kr!|S |}|t|8 }n| t| j }|dur<t| j|nd}| j| j| j	|| j
\| _ | _
d| _|dur[| |}n| t| j }|rk|| }|S |}|S )zzRead `n` or all contents of self.buff or file.

        Returns:
            result: Data from self.buff or file.
        Nr   )r   r|   r   r  maxr   r+   r@   r   rA   r   )r   nrz   chunkr  r   r   r   r@   \  s,   
z
BFile.readc                 C   s   | j std| jrtd| jr/| js#| j| j|| j d| _dS | j	| j|| j dS | j
du rA| jr9dnd}t|| _
| j
| dS )zqWrite contents to file.

        Args:
            file_content: Contents waiting to be written to file.
        zFile not opened in write modeFile already closedTNzw+bzw+)r   r    r   r   r   r+   rQ   r   rA   r\   r   tempfileTemporaryFile)r   rR   rB   r   r   r   rQ     s   

zBFile.writec                 C   s   d }	 | j s| d}|r|d dks| j s|S | j st nJ| j d| j}|dkrA| |d | j }|r=|| }|S |}|S | t| j d | j }|rT|| n|}|rc|d dksa| j sc|S | j sit q)NTr   
)r   r@   r   r&   r   r|   )r   liner*   r  r   r   r   r     s.   
zBFile.__next__c                 C   s   |   S r   )r   r   r   r   r   next  rp   z
BFile.nextc                 C   sz   | j rtd| js7| jdur9| j  | jd | j }|dur;| j| j	|| j
 | jt| dS dS dS dS )zFlush data to disk.
        r  Nr   )r   r    r   r   flushrI   r@   r+   rQ   r   rA   r|   )r   r  r   r   r   r    s   


zBFile.flushc                 C   sn   t | jtrz| jj| jd| jdd W n	 ty   Y nw |   | jd ur2| j	  d | _d| _
d| _d S )Nr   T)r   F)
isinstancer+   r%   r\   r   rA   	Exceptionr  r   r   r   r   r   r   r   r   r     s   



zBFile.closerc   r   )r-   r.   r/   r   r   r   r   r9   r  rE   r@   rQ   r   r  r  r   r   r   r   r   r   )  s    

.r   c                 C      t | | S r   )r!   r,   r3   r4   r   r   r   r3        r3   c                 C   r  r   )r!   r,   r5   r4   r   r   r   r5     r  r5   c                 G   s   t | j| g|R  S r   )r!   r,   r6   r7   r   r   r   r6     s   r6   c                 C   r  r   )r!   r,   ra   )rb   r   r   r   ra     r  ra   )#r   r   r2   r  r   r   Z	hdfs.utilr   r   ImportErrorZ baidubce.services.bos.bos_clientr   Zbaidubcer   Z!baidubce.bce_client_configurationr   Zbaidubce.auth.bce_credentialsr   r$   r   r   objectr   r!   r0   r   r"   r   r   r%   r   r3   r5   r6   ra   r   r   r   r   <module>   sN   #>7:   6