One Hat Cyber Team
Your IP :
216.73.216.183
Server IP :
23.137.84.82
Server :
Linux srv25.usacloudserver.us 5.14.0-570.39.1.el9_6.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 4 05:08:52 EDT 2025 x86_64
Server Software :
LiteSpeed
PHP Version :
8.1.33
Buat File
|
Buat Folder
Eksekusi
Dir :
~
/
lib64
/
python3.9
/
site-packages
/
borg
/
__pycache__
/
Edit File:
archive.cpython-39.opt-1.pyc
a HZ�h�� � @ s d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm Z d dl m Z m Z d dl mZ d dlmZ d dlmZ d dlmZmZ d d lmZ d d lmZmZmZm Z d d l m Z e � Z d d l m Z d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z'm(Z( d dl)m*Z*m+Z+ d dl,T d dl-m.Z/ d dl0m1Z1m2Z2m3Z3 d dl4m5Z5 d dl4m6Z6 d dl4m7Z7m8Z8m9Z9 d dl4m:Z:m.Z.m;Z; d dl<m=Z=m>Z>m?Z?m@Z@ d dl4mAZAmBZB d dl4mCZCmDZDmEZEmFZFmGZG d d l4mHZHmIZImJZJmKZK d d l4mLZL d d l4mMZM d d l4mNZN d d l4mOZOmPZPmQZQ d d!l4mRZRmSZSmTZT d d"l4mUZU d d#l4mVZV d d$l4mWZW d d%l4mXZX d d&lYmZZZ d d'l[m\Z\m]Z]m^Z^ d d(l_m`Z`maZambZb d d)l<mcZcmdZdmeZemfZfmgZgmhZh d d*limjZj d d+lkmlZlmmZm ened,�ZoG d-d.� d.�Zpd/d0� ZqG d1d2� d2er�ZsG d3d4� d4er�ZtG d5d6� d6�Zueu� Zvd7d8� Zwd9d:� Zxe dddd;d<d=�d>d?��ZyG d@dA� dA�ZzG dBdC� dC�Z{G dDdE� dEe{�Z|ddd d dF�dGdH�Z}G dIdJ� dJ�Z~G dKdL� dL�ZeZdMdNdO� dP�Z�dQdR� Z�G dSdT� dT�Z�G dUdV� dV�Z�G dWdX� dX�Z�dYdZ� Z�G d[d\� d\�Z�G d]d^� d^�Z�G d_d`� d`�Z�dS )a� N)� OrderedDict)�contextmanager)�timezone� timedelta)�partial)�getuser)�BytesIO)�groupby� zip_longest)�get_terminal_size� )�is_win32�is_linux� is_freebsd� is_darwin)� create_logger)�xattr)� get_chunker�Chunk��ChunkListEntry)� key_factory�UnsupportedPayloadError)� Compressor�CompressionSpec)�*)�IntegrityError)� ChunkIndex�ChunkIndexEntry�CacheSynchronizer)�Manifest�� hardlinkable)�ChunkIteratorFileWrapper�normalize_chunker_params� open_item)�Errorr �set_ec)�uid2user�user2uid� gid2group� group2gid)�parse_timestamp� to_localtime)�OutputTimestamp�format_timedelta�format_file_size� file_status�FileSize)� safe_encode� safe_decode�make_path_safe�remove_surrogates)� StableDict)� bin_to_hex)�safe_ns)�ellipsis_truncate�ProgressIndicatorPercent� log_multi)�os_open� flags_normal� flags_dir)�os_stat)�msgpack)�sig_int)�utcnow)�LRUCache)�PathPrefixPattern�FnmatchPattern� IECommand)�Item� ArchiveItem�ItemDiff)�acl_get�acl_set� set_flags� get_flags�swidth�hostname)�cache_if_remote)� Repository�LIST_SCAN_LIMIT�linkc @ s� e Zd Zd dd�Zd dd�Zdd� ZdZd d � Zd d � Zd d� Z dd� Z e dd� �Z e dd� �Ze dd� �Ze dd� �Zd dd�ZdS ) � StatisticsFc C sF || _ || _d | _ | _ | _| _d | _ | _ | _| _ d| _ d S �Nr ) � output_json�iec�osize�csize�usize�nfiles� osize_parts� csize_parts� usize_parts� nfiles_parts� last_progress)�selfrW rX � rc �2/usr/lib64/python3.9/site-packages/borg/archive.py�__init__: s zStatistics.__init__c C sf |s4| j |7 _ | j|7 _|rb| j|7 _n.| j|7 _| j|7 _|rb| j|7 _d S �N)rY rZ r[ r] r^ r_ )rb �sizerZ �unique�partrc rc rd �updateA s zStatistics.updatec C s� t |t�std��t| j| j�}| j|j |_| j|j |_| j|j |_| j|j |_| j |j |_ | j |j |_ | j |j |_ | j |j |_ |S )Nz can only add Statistics objects) � isinstancerU � TypeErrorrW rX rY rZ r[ r\ r] r^ r_ r` )rb �other�statsrc rc rd �__add__M s zStatistics.__add__zO{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s}c C s | j j| dd�S )Nz This archive:)rn Zlabel)�summary�format�rb rc rc rd �__str__] s zStatistics.__str__c C s dj t| �jt| �| d�S )NzF<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>)�cls�hashrb )rq �type�__name__�idrr rc rc rd �__repr__` s �zStatistics.__repr__c C s4 t | j| jd�t | j| jd�t | j| jd�| jd�S )N�rX )Z original_sizeZcompressed_sizeZdeduplicated_sizer\ )r2 rY rX rZ r[ r\ rr rc rc rd �as_dictd s �zStatistics.as_dictc C s | j | j| j| j| j| jd�S )N�rg rZ r\ � size_partsr^ r` �rY rZ r\ r] r^ r` rr rc rc rd � as_raw_dictl s �zStatistics.as_raw_dictc K sF | � }|d |_ |d |_|d |_|d |_|d |_|d |_|S )Nrg rZ r\ r} r^ r` r~ )rt �kwrb rc rc rd � from_raw_dictv s zStatistics.from_raw_dictc C s t | j| jd�S �Nrz )r0 rY rX rr rc rc rd � osize_fmt� s zStatistics.osize_fmtc C s t | j| jd�S r� )r0 r[ rX rr rc rc rd � usize_fmt� s zStatistics.usize_fmtc C s t | j| jd�S r� )r0 rZ rX rr rc rc rd � csize_fmt� s zStatistics.csize_fmtNc C s t �� }|d u s || j |k�r|| _| jrz|sP| �� }t|rD|jnd�|d< ni }|�t � � d|d�� t� |�}d}nrt � \} } |s�d� | �}|r�t|j�nd} | t |� } | dk r�d}| t |� } | dkr�|t | | �7 }nd | }d }t|||p�tjd d � d S ) N� �pathZarchive_progress)�timerv �finished� z={0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N � � � � T)�end�file�flush)r� � monotonicra rW r{ r6 r� rj �json�dumpsr rq rO r: �print�sys�stderr) rb �item�final�stream�dtZnow�data�msgr� �columns�linesr� �spacerc rc rd � show_progress� s8 � zStatistics.show_progress)FF)F)NFNN)rw � __module__� __qualname__re rj ro rp rs ry r{ r � classmethodr� �propertyr� r� r� r� rc rc rc rd rU 8 s" rU c C s t �| �p t �| �p t �| �S rf )�stat�S_ISBLK�S_ISCHR�S_ISFIFO)�moderc rc rd � is_special� s r� c @ s e Zd ZdZdS )� BackupErrorzY Exception raised for non-OSError-based exceptions while accessing backup files. N�rw r� r� �__doc__rc rc rc rd r� � s r� c @ s e Zd ZdZdd� Zdd� ZdS )� BackupOSErrora� Wrapper for OSError raised while accessing backup files. Borg does different kinds of IO, and IO failures have different consequences. This wrapper represents failures of input file or extraction IO. These are non-critical and are only reported (exit code = 1, warning). Any unwrapped IO error is critical and aborts execution (for example repository IO failure). c C s( || _ || _|j| _|j| _|j| _d S rf )�op�os_error�errno�strerror�filename)rb r� r� rc rc rd re � s zBackupOSError.__init__c C s& | j r| j � d| j� �S t| j�S d S )N�: )r� r� �strrr rc rc rd rs � s zBackupOSError.__str__N)rw r� r� r� re rs rc rc rc rd r� � s r� c @ s* e Zd ZdZd dd�Zdd� Zdd� ZdS ) �BackupIOr� c C s || _ | S rf )r� )rb r� rc rc rd �__call__� s zBackupIO.__call__c C s d S rf rc rr rc rc rd � __enter__� s zBackupIO.__enter__c C s |r t |t�r t| j|�|�d S rf )� issubclass�OSErrorr� r� )rb �exc_typeZexc_valZexc_tbrc rc rd �__exit__� s zBackupIO.__exit__N)r� )rw r� r� r� r� r� r� rc rc rc rd r� � s r� c c sd dt _t �> z t| �}W n ty8 Y W d � d S 0 W d � n1 sN0 Y |V qd S )N�read)� backup_ior� �next� StopIteration)�iteratorr� rc rc rd �backup_io_iter� s 2r� c C s8 t �| j�t �|j�kr td��| j|jkr4td��|S )aX this checks for some race conditions between the first filename-based stat() we did before dispatching to the (hopefully correct) file type backup handler and the (hopefully) fd-based fstat() we did in the handler. if there is a problematic difference (e.g. file type changed), we rather skip the file than being tricked into a security problem. such races should only happen if: - we are backing up a live filesystem (no snapshot, not inactive) - if files change due to normal fs activity at an unfortunate time - if somebody is doing an attack against us z1file type changed (race condition), skipping filez2file inode changed (race condition), skipping file)r� �S_IFMT�st_moder� �st_ino)Zst_oldZst_currrc rc rd �stat_update_check� s r� F�open)r� � parent_fd�name�noatimer� c c sp t |��" t|||| |d�}W d � n1 s00 Y z |V W |d urlt�|� n|d urjt�|� 0 d S )N�r� r� r� �flagsr� )r� r= �os�close)r� r� r� r� r� r� �fdrc rc rd �OsOpen s 0 �r� c @ s( e Zd Zdd� Zd dd�Zd dd�ZdS ) �DownloadPipelinec C s || _ || _d S rf )� repository�key)rb r� r� rc rc rd re s zDownloadPipeline.__init__NFc # sN �fdd�}t � }tjdd�}��|�D �] } |�| � dd� |D �} | D ] } d| v rJdd� | jD �| _qJ� r�� fd d�| D �} |�r6� �r|�r| D ]|} t| j�r�| �d �} | d u r�d| v r�|| j� | �d d �r�|� | j � q�| |vr�|| \} }| d u�r|| � |� | � q�n | D ]} d| v �r|| j� �q| D ] } | V �q:q(d S )a� Return iterator of items. *ids* is a chunk ID list of an item stream. *filter* is a callable to decide whether an item will be yielded. *preload* preloads the data chunks of every yielded item. Warning: if *preload* is True then all data chunks of every yielded item have to be retrieved, otherwise preloaded chunks will accumulate in RemoteRepository and create a memory leak. c s � j �dd� | D �� d S )Nc S s g | ] }|j �qS rc �rx ��.0�crc rc rd � <listcomp>& � zBDownloadPipeline.unpack_many.<locals>._preload.<locals>.<listcomp>)r� �preload)�chunksrr rc rd �_preload% s z.DownloadPipeline.unpack_many.<locals>._preloadF�Zuse_listc S s g | ]}t |d ��qS )�Z internal_dict)rH �r� r� rc rc rd r� , r� z0DownloadPipeline.unpack_many.<locals>.<listcomp>r� c S s g | ] }t |� �qS rc r )r� �erc rc rd r� / r� c s g | ]}� |�r|�qS rc rc r� ��filterrc rd r� 2 r� �sourceN�hardlink_masterT) �setrA �Unpacker� fetch_many�feedr� r" r� �get�addr� )rb �idsr� �partial_extractr� �hardlink_mastersr� Zmasters_preloaded�unpackerr� �itemsr� r� r� �_rc �r� rb rd � unpack_many s> z DownloadPipeline.unpack_manyc c s4 t || jj||d��D ]\}}| j�||�V qd S )N�� is_preloaded)�zipr� �get_manyr� �decrypt)rb r� r� �id_r� rc rc rd r� T s zDownloadPipeline.fetch_many)NFFN)F)rw r� r� re r� r� rc rc rc rd r� s 9r� c @ s> e Zd ZdZefdd�Zdd� Zdd� Zdd d �Zd d � Z d S )� ChunkBufferi � c C s8 t � | _t�� | _g | _|| _t|| jjdd��| _ d S )NF�Zseed�sparse) r �bufferrA ZPacker�packerr� r� r � chunk_seed�chunker)rb r� �chunker_paramsrc rc rd re \ s zChunkBuffer.__init__c C s, | j �| j�|�� �� | �� r(| �� d S rf )r� �writer� �packr{ �is_fullr� )rb r� rc rc rd r� c s zChunkBuffer.addc C s t �d S rf )�NotImplementedError)rb �chunkrc rc rd � write_chunkh s zChunkBuffer.write_chunkFc C s� | j �� dkrd S | j �d� g }| j�| j �D ]X}|jd }|tkrRt|j�}n,|t t fv rrt d |jd � }n t d| ��|� |� q0| j �d� | j �d� |s�t|�dkr�d nd}|d |� D ]}| j� | �|�� q�|dkr�| j �|d � d S )Nr � allocationrg z,chunk allocation has unsupported value of %rr ���)r� �tell�seekr �chunkify�meta�CH_DATA�bytesr� �CH_ALLOC�CH_HOLE�zeros� ValueError�append�truncate�lenr� r r )rb r� r� r Zallocr� r� rc rc rd r� k s& zChunkBuffer.flushc C s | j �� | jkS rf )r� r � BUFFER_SIZErr rc rc rd r � s zChunkBuffer.is_fullN)F) rw r� r� r �ITEMS_CHUNKER_PARAMSre r� r r� r rc rc rc rd r� Y s r� c s( e Zd Zef� fdd� Zdd� Z� ZS )�CacheChunkBufferc s t � �||� || _|| _d S rf )�superre �cachern )rb r r� rn r �� __class__rc rd re � s zCacheChunkBuffer.__init__c C s8 | j j| j�|�|| jdd�\}}}| j jjdd� |S �NF��wait)r � add_chunkr� �id_hashrn r� �async_response)rb r r� r� rc rc rd r � s $z CacheChunkBuffer.write_chunk)rw r� r� r re r � __classcell__rc rc r rd r � s r )� uid_forced� gid_forced� uid_default� gid_defaultc C s� |d ur|}n0|rd nt | j�}|d u r.| jn|}|dk r>|}|d urL|}n0|rTd nt| j�}|d u rl| jn|}|dk r||}||fS rV )r) �user�uidr+ �group�gid)r� �numericr% r& r' r( r* r, rc rc rd �get_item_uid_gid� s r. c @ sX e Zd ZG dd� de�ZG dd� de�ZG dd� de�Zddd dddddddeddddddfd d �Zd d � Z dd� Z e dd� �Z e dd� �Z e dd� �Ze dd� �Ze dd� �Zdd� Zd d � Zd d � Zd>d d!�Zd?d"d#�Zd@d%d&�Zd'd(� ZdAd)d*�ZdBd+d,�ZdCd-d.�Zed/d0� �Z dDd2d3�Z dEd4d5�Z d6d7� Z d8d9� Z dFd:d;�Z!e"dGd<d=��Z#dS )H�Archivec @ s e Zd ZdZdS )zArchive.DoesNotExistzArchive {} does not existNr� rc rc rc rd � DoesNotExist� s r0 c @ s e Zd ZdZdS )zArchive.AlreadyExistszArchive {} already existsNr� rc rc rc rd � AlreadyExists� s r1 c @ s e Zd ZdZdS )z+Archive.IncompatibleFilesystemEncodingErrorzrFailed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.Nr� rc rc rc rd �#IncompatibleFilesystemEncodingError� s r2 NF� c C s� t �� | _|| _|| _|| _|| _i | _t||d�| _ || _ || _ || _ || _ d | _d| _|| _|| _| | _| | _| | _| | _| | _|d u r�t� }t�� }|| _|| _|| _ |d u r�t� }|| _ || _ t | j| j�| _ || _!| j!�rPt"| j| j| j �| _#||j$v �r| �%|��d}d�&||�r(d| �p*d�| _'| j'|j$v�rB�q~|d7 }�qn.| jj$�(|�}|d u �rr| �)|��| �*|j+� d S )N�rW rX Fr z{}.checkpoint{}z.%dr� r ),r� �getcwd�cwdr� r� r �manifest� hard_linksrU rn rX r� r� Zname_in_manifest�comment� tam_verified�checkpoint_interval� numeric_idsr� �noctime�noflags�noacls�noxattrsrC r� r� r �start�start_monotonicr� �consider_part_filesr� �pipeline�creater � items_buffer�archivesr1 rq �checkpoint_namer� r0 �loadrx )rb r� r� r7 r� r rE r; r< r� r= r> r? r@ �progressr rA rB r� rC �log_jsonrX �i�inforc rc rd re � sZ zArchive.__init__c C sN | j �|| j�|��}| j j|dd�\}| _}t|d�}|jdkrJtd��|S )NT�Zforce_tam_not_requiredr� r � Unknown archive metadata version) r� r� r� r� �unpack_and_verify_archiver: rI �version� Exception)rb rx r� �archiver� �metadatarc rc rd � _load_meta� s zArchive._load_metac C sH || _ | �| j �| _dd� | jjD �| j_| jj| _| j�dd�| _d S )Nc S s g | ] }t |��qS rc �r4 �r� �argrc rc rd r� � r� z Archive.load.<locals>.<listcomp>r9 r� )rx rU rT �cmdliner� r� r9 )rb rx rc rc rd rI � s z Archive.loadc C s | j j}t|�S )z,Timestamp of archive creation (start) in UTC)rT r� r, �rb �tsrc rc rd r[ � s z Archive.tsc C s | j �d�p| j j}t|�S )z*Timestamp of archive creation (end) in UTC�time_end)rT r� r� r, rZ rc rc rd �ts_end s zArchive.ts_endc C s t | j�S rf )r8 rx rr rc rc rd �fpr s z Archive.fprc C s t | j| j �S rf )r/ r� rA rr rc rc rd �duration s zArchive.durationc C s t | j| j �S rf )r/ r] r[ rr rc rc rd �duration_from_meta s zArchive.duration_from_metac C s� | j r.| j}| jjtjd�}| jjtjd�}n| �| j�}| j }| j }| j | j t |�t |�|| �� |�� d| jj| j jt id�}| j r�tj|d< nL| j�d�}|d ur�t|�nd}|�| jj| jj| jj | j�dd�|d�� |S ) N�ZtzinfoZmax_archive_size)r� rx rA r� r_ rn Zlimits� command_liner r� r9 )rb rP �usernamer9 r ) rE rn rA �replacer �utcr� � calc_statsr r[ r] r� r^ r. Z total_secondsr{ r� rx rZ � MAX_DATA_SIZEr� �argvrT r� r$ rj rY rP rc )rb rn rA r� rM Zcprc rc rd rM s8 �� �z Archive.infoc C sL dj | t| jjtjd��t| jjtjd��| jj| j j t | j j �� d�S )Nz�Repository: {location} Archive name: {0.name} Archive fingerprint: {0.fpr} Time (start): {start} Time (end): {end} Duration: {0.duration} Number of files: {0.stats.nfiles} Utilization of max. archive size: {csize_max:.0%} ra )rA r� Z csize_max�location)rq r. rA rd r re r� r r� rx rZ rg r� Z _locationZcanonical_pathrr rc rc rd rs 5 s �zArchive.__str__c C s d| j S )Nz Archive(%r))r� rr rc rc rd ry G s zArchive.__repr__c C s" | j sd|v rdS |r ||�S dS )Nri FT)rC )rb r� r� rc rc rd � item_filterJ s zArchive.item_filterc # s4 �j j�jj|||� �fdd�d�D ] }|V q$d S )Nc s �� | � �S rf )rj �r� r� rc rd �<lambda>V r� z$Archive.iter_items.<locals>.<lambda>)r� r� r� r� )rD r� rT r� )rb r� r� r� r� r� rc r� rd � iter_itemsP s � zArchive.iter_itemsTc C s6 |r&| j r&|d u r| j}|j |dd� | j�|� d S )N皙�����?�r� r� )r� rn rF r� )rb r� r� rn rc rc rd �add_itemY s zArchive.add_itemc C s. | � | j� | jj| j= | j�| j| j� d S rf )�saverH r7 rG r � chunk_decrefrx rn rr rc rc rd �write_checkpoint` s zArchive.write_checkpointc C s� |p| j }|| jjv r | �|��| jjdd� tt�� | j d�}|d u rZt � }|| }n || }|}|| _ || _ d||p|d| jj tjtt� |�t�|�t�| jd� } |p�| j}| �|j|j|j|j|j|j d�� | �|p�i � t | �} | j j | � � dd �} | j �!| �| _"z| j#�$| j"| | j� W nH t%�yl } z.t&| �} d | v �rVt'd | ��n� W Y d } ~ n d } ~ 0 0 | j(j)dd �d u�r��qn| j"| jf| jj|< | j�*� | j(j+d d� | j#�+� d S )NT�r� )Zsecondsr r� ) rQ r� r9 r� rY rP rc r� r\ r r| � archive��contextzMore than allowed put dataz#%s - archive too big (issue #1473)!r F)�compact),r� r7 rG r1 rF r� r r� r� rB rC rA r� r� r� rh rP r �strftimeZ ISO_FORMATr rn rj rY rZ r\ r] r^ r` rI r� � pack_and_authenticate_metadatar{ r" rx r r! r r� r& r� r# r �commit) rb r� r9 � timestamprn �additional_metadatar_ r� rA rT r� �err�err_msgrc rc rd rq e sb � � z Archive.savec C sn | j �d�d u}z$|rt�tjf i |j| j ��}W n4 tyh | j||d�}|sd|�� |j| j< Y n0 |S )Nr\ )� want_unique) rT r� �KeyErrorrU r� Z pre12_metar^ � _calc_statsr )rb r r� �have_borg12_metarn rc rc rd rf � s zArchive.calc_statsc s� | j �d�d u}|r |s d}n�� �fdd�}t� � t� �}|| j� | j�dd�}tt| j j �d| dd �}t | j j | j � | j j ��D ]4\} } |j d d � || � | j�| | �} |�| � q�� ��j�d }|�� t| jd �} || _|�s:| j�r|j| _|j| _|j | _ n*|j|j | _|j|j | _|j |j | _ n^| j�rz| j j!| j j | _| j j | j j" | _| j j | j j | _ n | j j| _| j j"| _| j j | _ | S )Nr\ r c s" �j | }� �| d|j|j� d S )Nr )r� r� rg rZ )rx �entry�Z archive_indexr rc rd r� � s z Archive._calc_stats.<locals>.add�%z%%z4Calculating statistics for archive %s ... %%3.0f%%%%zarchive.calc_stats��totalr� �msgidr )�increase� rz )#rT r� r r rx r� rd r; r r� r� r� r� �showr� r� r� Z stats_againstr� �finishrU rX r[ rC Znum_files_totalsr\ Z size_totalsrY Z csize_totalsrZ Znum_files_partsr} r^ r` rg ) rb r r� r� Z unique_csizer� �syncZarch_name_escd�pirx r r� rn rc r� rd r� � sJ �" zArchive._calc_statsc c s� d}d|v r�t jj|g|j�t j�|d � �R � }|�|jd |f�\} } | r�tr�td�� t � | |� d}W d � q�1 s~0 Y n| d ur�| |_ |V |s�|r�tr�d |f||�d�p�|< n d S )NFr� rT T) r� r� �joinr� �split�sepr� �has_linkr� rT r� ) rb �destr� r� �stripped_components� original_pathr� � hardlink_setr� r� Z link_targetrc rc rd �extract_helper� s & $zArchive.extract_helperr c C s� |pi }d|v } |s|r�d|v r�d} | j jdd� |jD �dd�D ]@} | rb| jt| �t|j�gd� |rttjj � | � | t| �7 } q@|r�tjj � � d |v r�|j } | | kr�t d �| | ���| r�t d ��d S |p�|j}| j}|j�d �r�td��tj�||j�}z6tj|dd�}t�|j��r&t�|� n t�|� W n8 t�yX | �|t�� �d �Y n t �yj Y n0 dd� }|j }t� |��r�t d�� ||� W d � n1 �s�0 Y | � ||||||����}|�r�W d � d S t d�� t!|d�}W d � n1 �s0 Y |�� dd� |jD �}| j j|dd�D ]~} | �rf| jt| �t|j�gd� t d��> |�r�t"�| ��r�|�#t| �d� n |� | � W d � n1 �s�0 Y �qBt d��B |�$� }} |�%|� |� � | j&|||�'� d� W d � n1 �s0 Y W d � n1 �s40 Y d |v �rh|j } | | k�rht d �| | ���| �rvt d ��W d � n1 �s�0 Y d S t ��� t�|��r�||� tj�(|��s�t�)|� |�rt| �&||� �n�t�*|��rJ||� |j+}zt�,||� W n& t�y4 | �|t�� �d �Y n0 | j&||dd� �n*t�-|��r�||� | � ||||||��H}|�r�W d � W d � d S t�.|� | �&||� W d � n1 �s�0 Y n�t�/|��s�t�0|��rf||� | � ||||||��P}|�r&W d � W d � d S t�1||j |j2� | �&||� W d � n1 �sZ0 Y ntd |j ��W d � n1 �s�0 Y d S ) a� Extract archive item. :param item: the item to extract :param restore_attrs: restore file attributes :param dry_run: do not write any data :param stdout: write extracted data to stdout :param sparse: write sparse files (chunk-granularity, independent of the original being sparse) :param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly :param stripped_components: stripped leading path components to correct hard link extraction :param original_path: 'path' key as stored in archive :param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes) �chunks_healthyr� r c S s g | ] }|j �qS rc r� r� rc rc rd r� r� z(Archive.extract_item.<locals>.<listcomp>Tr� )r� rM rg z4Size inconsistency detected: size {}, chunks size {}zDFile has damaged (all-zero) chunks. Try running borg check --repair.N)�/z../z!Path should be relative and localF��follow_symlinksc S s& t j�| �}t j�|�s"t �|� d S rf )r� r� �dirname�exists�makedirs)r� Z parent_dirrc rc rd � make_parent/ s z)Archive.extract_item.<locals>.make_parentr� r� �wbc S s g | ] }|j �qS rc r� r� rc rc rd r� ? r� r r Ztruncate_and_attrs�r� )�symlinkz Unknown archive item type %r)3rD r� r� r� r r6 r� r� �stdoutr� r r� rg r� rq r6 � startswithrR r� r� r� �S_ISDIRr� �rmdir�unlink�UnicodeEncodeErrorr2 �getfilesystemencodingr� r� �S_ISREGr� r� r� r r r r � restore_attrs�filenor� �mkdir�S_ISLNKr� r� r� �mkfifor� r� �mknod�rdev)rb r� r� �dry_runr� r� r� r� r� r� Zhas_damaged_chunks�item_chunks_sizer� � item_sizer� r� �str� r� r� r� r� �posr� rc rc rd � extract_item� s� � (� * . T �( � .� .zArchive.extract_itemc C s� dt _t|| jd�\}}t�s�z*|r4t�|||� ntj|||dd� W n tyZ Y n0 |rpt� ||j � n<ztj ||j dd� W n$ t y� |s�t� ||j � Y n0 | j s�t||| j|d� | js�tj|p�||�di �dd�}|r�tt� |j}d|v �r |j} n|} d|v �rj|j} z6|�r<tj|d | | fd � ntj|d | | fdd � W n t�yh Y n0 z6|�r�tj|d | |fd � ntj|d | |fdd � W n t�y� Y n0 | j�s�d |v �r�zt||j|d� W n t�y� Y n0 d S ) zv Restore filesystem attributes on *path* (*fd*) from *item*. Does not access the repository. �attrs)r- Fr� r� �xattrs�atime� birthtimeN)�ns)r� r� �bsdflags) r� r� r. r< r r� �fchown�chownr� �fchmodr� �chmodr r? rL r@ r Zset_allr� r' Z EXIT_WARNING�mtimer� r� �utimer> rM r� ) rb r� r� r� r� r* r, �warningr� r� r� rc rc rd r� y sZ zArchive.restore_attrsc C sz | � | j�}t|||� | jj|�� dd�}| j�|�}| j�||| j � ||j f| j j | j < | j�| j| j � || _d S )Nru rv )rU rx �setattrr� rz r{ r" r r! rn r� r7 rG r� rr )rb r� �valuerT r� Znew_idrc rc rd �set_meta� s zArchive.set_metac C s< || j jv r| �|��| j}|| _| �d|� | j j|= d S )Nr� )r7 rG r1 r� r� )rb r� Zoldnamerc rc rd �rename� s zArchive.renamec s� G dd� dt �� t� �d����fdd� �d� ��fdd� }d��ztjdd �}�jj}tt|�d d d �}tt |�j � |���D ]�\}\} } |r�|� |� �j �| | �} |�| � || |� zP|D ]F} t| d �} d| v rj o�d| v } | jD ]\} }}|| || d� q�q�W q� ttf�y4 �dk�r,� d�Y q�0 q�|�rF|�� W n, tjtjf�yt �dk�rl� d�Y n0 |�j|� �jj�j = �dd�d u�r��q���r�t � d� t � d� d S )Nc @ s e Zd ZdZdS )z(Archive.delete.<locals>.ChunksIndexErrorzUChunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.Nr� rc rc rc rd �ChunksIndexError� s r� Tc s< z�j j| d�W S tjy6 �dkr*� d� � Y S 0 d S )Nr r T)r� r# rR �ObjectNotFoundr )�error�exception_ignored�forcedrb rc rd �fetch_async_response� s z,Archive.delete.<locals>.fetch_async_responseFc sJ z�j j| |d|d� W n" ty: t| �}� |��Y n 0 �dd� d S )NF)r ri r )r rr r� r8 )rx rn ri �cid)r� r� rb rc rd rr � s z$Archive.delete.<locals>.chunk_decrefr� z Decrementing references %3.0f%%zarchive.deleter� r� r� ri )ri r r zAforced deletion succeeded, but the deleted archive was corrupted.z2borg check --repair is required to free all space.)T)F) r& �objectrA r� rT r� r; r � enumerater� r� r� r� r� r� r� rH rC r� rl r r� �UnpackExceptionrR r� rx r7 rG r� �loggerr� )rb rn rJ r� rr r� Z items_idsr� rL Zitems_idr� r� ri �chunk_idrg rZ rc )r� r� r� r� r� rb rd �delete� sN " zArchive.deletec # s� �fdd��dd� ���fdd�� dd� �� �����fd d �}��� fd d �}t � }t � }g �i �t� �� fd d����� fdd���D ]�\} } | r�| r�| j| jkr�|| | �s�| j|| | �fV q�| �r|�| jd�} | r�|| | �s�| j|| | �fV n | || j< | r�|�| jd�} | �r<|| | ��sF| j|| | �fV q�| || j< q�|�� D ]0} | j} t�| �}� || � | ||| �fV �qP|�� D ]0}|j} t�| �}� ||� | |||�fV �q��D ] \} } | j|| | �fV �q�dS )a% Yields tuples with a path and an ItemDiff instance describing changes/indicating equality. :param matcher: PatternMatcher class to restrict results to only matching paths. :param can_compare_chunk_ids: Whether --chunker-params are the same for both archives. c s d| vp t | j� p | j� v S �Nr� )r" r� r� rk )r� rc rd �hardlink_master_seen# s z;Archive.compare_archives_iter.<locals>.hardlink_master_seenc S s | � dd�o d| vo t| j�S �Nr� Tr� )r� r" r� rk rc rc rd �is_hardlink_master& s z9Archive.compare_archives_iter.<locals>.is_hardlink_masterc s" �| �s�|�r | |f� | j < d S rf �r� ��item1�item2)r� r� rc rd �update_hardlink_masters) s z>Archive.compare_archives_iter.<locals>.update_hardlink_mastersc S s t | j�o| �d�|v S r� �r" r� r� )r� r� rc rc rd �has_hardlink_master- s z:Archive.compare_archives_iter.<locals>.has_hardlink_masterc sx �| ��r�| j d } �|��r0�|j d }t| |� j�dd� | �dg �D ���j�dd� |�dg �D ����d�S )Nr r c S s g | ] }|j �qS rc r� r� rc rc rd r� 6 r� zHArchive.compare_archives_iter.<locals>.compare_items.<locals>.<listcomp>r� c S s g | ] }|j �qS rc r� r� rc rc rd r� 7 r� )�can_compare_chunk_ids� content_only)r� rJ rD r� r� r� )�archive1�archive2r� r� r� r� rc rd � compare_items0 s �z4Archive.compare_archives_iter.<locals>.compare_itemsc s4 �| |� �| � p �|� }|r0� � | |f� |S )zQAdds item tuple to deferred if necessary and returns True, if items were deferred)r )r� r� Zdefer)�deferredr� r� rc rd �defer_if_necessary; s z9Archive.compare_archives_iter.<locals>.defer_if_necessaryc s � � | j�S rf ��matchr� rk ��matcherrc rd rl I r� z/Archive.compare_archives_iter.<locals>.<lambda>c s � � | j�S rf r� rk r� rc rd rl J r� N)r r rm r� �pop�valuesrH Zcreate_deleted)r� r� r� r� r� r� r� Zorphans_archive1Zorphans_archive2r� r� Zmatching_orphanZaddedr� Z deleted_itemZdeletedrc ) r� r� r� r� r� r� r� r� r� r� r� rd �compare_archives_iter sT � z Archive.compare_archives_iter)N)NFFN)TN)NNNNN)T)T)TFFFNr NN)FN)FF)NFF)$rw r� r� r& r0 r1 r2 �CHUNKER_PARAMSre rU rI r� r[ r] r^ r_ r` rM rs ry rj rm rp rs rq rf r� r r� r� r� r� r� r� � staticmethodr� rc rc rc rd r/ � sV � 6 " 9 0 � I Fr/ c @ s0 e Zd Zdd� Zdd� Zd dd�Zd dd �ZdS ) �MetadataCollectorc C s. || _ || _|| _|| _|| _|| _|| _d S rf )r� r= r< r> r? r@ � nobirthtime)rb r� r= r� r< r> r? r@ rc rc rd re q s zMetadataCollector.__init__c C s� t |j|j|jt|j�d�}| js0t|j�|d< | jsDt|j �|d< | j sjt |d�rjtt |j d ��|d< | jr�d |d< |d< n t|j�|d< t|j�|d< |S ) N)r� r* r, r� r� �ctime� st_birthtime� ʚ;r� r) r+ )�dictr� �st_uid�st_gidr9 � st_mtime_nsr� � st_atime_nsr= � st_ctime_nsr� �hasattr�intr� r< r( r* )rb r� r� rc rc rd �stat_simple_attrsz s" � z#MetadataCollector.stat_simple_attrsNc C s� i }| j sLtd�� t|||d�}W d � n1 s60 Y |rL||d< | js�td��" tj|pf|dd�}W d � n1 s�0 Y |r�t|�|d< | js�td��$ t|||| j |d� W d � n1 s�0 Y |S ) Nzextended stat (flags)r� r� zextended stat (xattrs)Fr� r� zextended stat (ACLs)) r> r� rN r@ r Zget_allr7 r? rK r< )rb r� r� r� r� r� r� rc rc rd �stat_ext_attrs� s , 0 2z MetadataCollector.stat_ext_attrsc C s$ | � |�}|�| j|||d�� |S )Nr� )r� rj r )rb r� r� r� r� rc rc rd � stat_attrs� s z MetadataCollector.stat_attrs)N)N)rw r� r� re r� r r rc rc rc rd r� p s r� � c C s d S rf rc )r� rc rc rd rl � r� rl )Zdisposec C s� | j d }|tkr"| j}||�}nh|ttfv r�| j d }tt�d |� }zt||f }W q� ty~ ||�}|t||f< Y q�0 nt d��||fS )Nr rg zunexpected allocation type) r r r� r r � memoryviewr �zero_chunk_idsr� r )r r" r r� r� rg rc rc rd � cached_hash� s r c @ s0 e Zd Zdd� Zdd� Zd dd�Zd d d �ZdS ) �ChunksProcessorc C s2 || _ || _|| _|| _|| _t�� | _|| _d S rf ) r� r rp rs r; r� r� �last_checkpoint� rechunkify)rb r� r rp rs r; r rc rc rd re � s zChunksProcessor.__init__c C st t |�� d�}t|j�}|j|d � |_|jddd� | jd| 7 _||_|d7 }| j|dd� | �� ||fS )Nr� T)�memorize� from_chunksz .borg_part_%dr F)r� ) rH r{ r r� �get_sizer� ri rp rs )rb r� � from_chunk�number�lengthrc rc rd �write_part_file� s z ChunksProcessor.write_part_fileFc C sv t o t �� }|s.|s.| jrnt�� | j | jkrn|r<t�d� | �|||�\}}t�� | _|rnt � � t�d� ||fS )Nz5checkpoint requested: starting checkpoint creation...z3checkpoint requested: finished checkpoint creation!) rB Zaction_triggeredr; r� r� r r� rM r Zaction_completed)rb r� r � part_numberr� Zsig_int_triggeredrc rc rd �maybe_checkpoint� s �� z ChunksProcessor.maybe_checkpointNc s� |s� ��fdd�}g |_ �jr,d|v r,|`d}d}|D ]<} |j �|| �� |r^�j|dd� �j|||dd �\}}q8|dkr�|j |d � r��j|||d d �\}}|j D ]} � j| j�| jd d � q�� j |d 7 _ d S ) Nc s8 t | �jj�\}}� j||�dd�}�jjjdd� |S r )r r� r" r! r r� r# )r r� r� � chunk_entry�r rb rn rc rd �chunk_processor� s z<ChunksProcessor.process_file_chunks.<locals>.chunk_processorr� r r rn ro F)r� T)rg ri ) r� r r� r r� r � chunk_increfrx rg r` ) rb r� r rn r� Z chunk_iterr r r r rc r rd �process_file_chunks� s$ z#ChunksProcessor.process_file_chunks)F)N)rw r� r� re r r r rc rc rc rd r � s r c @ sf e Zd Zdd�dd�Zeddd��Zdd � Zd d � Zd d � Zdd� Z dd� Z dd� Z e d�dd�Z dS )�FilesystemObjectProcessorsN��file_status_printerc C sh || _ || _|| _|| _|| _|| _| p.dd� | _i | _t| | d�| _ t � � | _ t ||j|d��| _d S )Nc W s d S rf rc ��argsrc rc rd rl r� z5FilesystemObjectProcessors.__init__.<locals>.<lambda>r4 r� )�metadata_collectorr r� rp r r� �print_file_statusr8 rU rn r� r5 r6 r r� r ) rb r r r� rp r r r� r� rK rX r rc rc rd re s z#FilesystemObjectProcessors.__init__Tc c s� t |�}t|d�}d}|o"|jdk}|rT| j�|j|jf�} | d urP| |_d}nd}||||fV | j|| j d� |r�|| j|j|jf< d S )Nr� Fr �hT�rn ) r5 rH �st_nlinkr8 r� r� �st_devr� rp rn ) rb r� r� �statusr" Z safe_pathr� r� � hardlinkedr� rc rc rd � create_helper& s z(FilesystemObjectProcessors.create_helperc C sZ | j ||ddd��4\}}}}|�| jj|||d�� |W d � S 1 sL0 Y d S )N�dFr! r� )r$ rj r r )rb r� r� r� r� r" r# r� rc rc rd �process_dir_with_fd: s z.FilesystemObjectProcessors.process_dir_with_fdc C s� | j ||ddd���\}}}}t|||tddd��z} | d urrtd�� t|t�| ��}W d � n1 sh0 Y |�| jj ||| d�� |W d � W d � S 1 s�0 Y W d � n1 s�0 Y d S ) Nr% Fr! TZdir_open)r� r� r� r� r� r� �fstatr� ) r$ r� r? r� r� r� r' rj r r � rb r� r� r� r� r� r" r# r� r� rc rc rd � process_dir? s � .z&FilesystemObjectProcessors.process_dirc C s� | � ||d���\}}}}t|||tdd���} td�� t|t�| ��}W d � n1 sZ0 Y |rl|rr||_|�| j j ||| d�� |W d � W d � S 1 s�0 Y W d � n1 s�0 Y d S )N�fTr� r' r� ) r$ r� r> r� r� r� r' r� rj r r r( rc rc rd � process_fifoJ s .z'FilesystemObjectProcessors.process_fifoc C s� | � |||���\}}}} td��& t|t|||dd��}W d � n1 sL0 Y |j|_|rf| rl||_|�| j� ||�� |W d � S 1 s�0 Y d S )Nr� F)r� r� r� r� ) r$ r� r� r@ �st_rdevr� r� rj r r ) rb r� r� r� r� Zdev_typer� r" r# r� rc rc rd � process_devT s 4z&FilesystemObjectProcessors.process_devc C s� | j ||ddd���\}}}}|d ur0|d ur0|n|} td�� tj| |d�} W d � n1 s`0 Y | |_|�| j�||�� |W d � S 1 s�0 Y d S )N�sFr! �readlink)�dir_fd)r$ r� r� r/ r� rj r r ) rb r� r� r� r� r� r" r# r� �fnamer� rc rc rd �process_symlink_ s ,z*FilesystemObjectProcessors.process_symlinkc C s� d}| � ||� d }t|�}|d u r0td| ��t|�} | d u rLtd| ��tt�� �d } t||d@ dB ||| || | | d� } | �| || j| j t | j � |��� | j dd � | j jd 7 _d | _| j| | jd � |S ) NrL zno such user: %szno such group: %sr� i�� i � ) r� r� r* r) r, r+ r� r� r� T�r r Fr )r r) r& r+ r� r� rH r rn r� r� r r r r\ r� rp ) rb r� r r� r� r) r+ r" r* r, �tr� rc rc rd � process_pipek s. �" z'FilesystemObjectProcessors.process_pipe)r� c s� �� ||d ����\}}} } t||||dd���h} td�� t|t�| ��}W d � n1 s^0 Y |��j�|�� t |j �} | r�t j t � |j�B |_| r�| �rR| s�ttj��j|��} �j�| �}� �| ||�\}}nd } }d\}}d }|d u�r8|D ] }� �|��s d} �qF�q � �fdd�|D �}d}n|�rBdnd }��||� d }| |_|d u�rn||_n�td ��4 ��|� �j�j t �j � d | ��� W d � n1 �s�0 Y t �r�d }nHtd �� t�| �}W d � n1 �s�0 Y | �o|j!|j!k}|�rd }| �sB|�sB� �"| ||dd� |jD �� �j j#d7 _#|��jj$||| d�� |j%dd� |W d � W d � S 1 �s�0 Y W d � n1 �s�0 Y d S )NTr� r' )FN�Mc s g | ]}� � |�j��qS rc )r rn )r� r� �r rb rc rd r� � r� z;FilesystemObjectProcessors.process_file.<locals>.<listcomp>�U�Ar� FZfstat2�Cc S s g | ] }|j �qS rc r� r� rc rc rd r� � r� r r� r3 )&r$ r� r� r� r� r' rj r r� r� r� r� �S_IFREG�S_IMODEr� r3 r� r� r6 r� r" Zfile_known_and_unchangedZ seen_chunkr r� r� r rn r� r� r r r r� Z memorize_filer\ r r )rb r� r� r� r� r r� r� r" r# r� r� Zis_special_fileZ hashed_pathZ path_hashZknownr� r� r� Zchanged_while_backupZst2rc r7 rd � process_file� sX . D * z'FilesystemObjectProcessors.process_file)NT)rw r� r� re r r$ r& r) r+ r- r2 r5 r> r= rc rc rc rd r s � r c @ sP e Zd Zdd�dd�Zeddd��Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )�TarfileObjectProcessorsNr c C sR || _ || _|| _|| _|| _| p(dd� | _t||d�| _t||j dd��| _ d S )Nc W s d S rf rc r rc rc rd rl � r� z2TarfileObjectProcessors.__init__.<locals>.<lambda>r4 Fr� ) r r� rp r r� r rU rn r r� r ) rb r r� rp r r r� rK rX r rc rc rd re � s z TarfileObjectProcessors.__init__c c sj t j�|j�}tt|�|j|B |j|j|j p.d |j p6d t t |j d ��d�}||fV | j|| jd� d S )Nr� )r� r� r* r, r) r+ r� r )r� r� �normpathr� rH r5 r� r* r, �uname�gnamer9 r� r� rp rn )rb �tarinfor" rv Znormalized_pathr� rc rc rd r$ � s � z%TarfileObjectProcessors.create_helperc C s: | � |||��\}}|W d � S 1 s,0 Y d S rf �r$ �rb rB r" rv r� rc rc rd r) � s z#TarfileObjectProcessors.process_dirc C s: | � |||��\}}|W d � S 1 s,0 Y d S rf rC rD rc rc rd r+ � s z$TarfileObjectProcessors.process_fifoc C sL | � |||��*\}}t�|j|j�|_|W d � S 1 s>0 Y d S rf )r$ r� �makedevZdevmajorZdevminorr� rD rc rc rd r- � s z#TarfileObjectProcessors.process_devc C sB | � |||�� \}}|j|_|W d � S 1 s40 Y d S rf )r$ Zlinknamer� rD rc rc rd � process_link� s z$TarfileObjectProcessors.process_linkc C s� | � |||��t\}}| �||j� d }|�|�}| �|| j| j| jt| j � |��� |j dd� | j j d7 _ |W d � S 1 s�0 Y d S )NTr3 r ) r$ r r� Z extractfiler r rn r� r� r r r r\ )rb rB r" rv �tarr� r� rc rc rd r= � s � z$TarfileObjectProcessors.process_file)NN) rw r� r� re r r$ r) r+ r- rF r= rc rc rc rd r> � s � r> c s� t | �}|dkrdS | d d@ dkr*d}n| d dkr<d}ndS ||krLdS | | d@ d kr^n| | d v rlndS | |d � � t� fd d �|D ��S )z1check if the data <d> looks like a msgpacked dictr F�� � r �� r� �� � )�� �� �� Nc 3 s | ]}� � |�V qd S rf )r� )r� �pattern�Zkey_serializedrc rd � <genexpr> r� z'valid_msgpacked_dict.<locals>.<genexpr>)r �any)r% Zkeys_serializedZd_lenZoffsrc rQ rd �valid_msgpacked_dict s" rT c s@ e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Z� Z S ) �RobustUnpackerzCA restartable/robust version of the streaming msgpack unpacker c s> t � �� dd� |D �| _|| _g | _d| _tjtd�| _ d S )Nc S s g | ]}t �|�� ��qS rc �rA Zpackb�encode�r� r� rc rc rd r� ! r� z+RobustUnpacker.__init__.<locals>.<listcomp>F�Z object_hook) r re � item_keys� validator�_buffered_data�_resyncrA r� r7 � _unpacker)rb r[ rZ r rc rd re s zRobustUnpacker.__init__c C s g | _ d| _d S )NT)r\ r] rr rc rc rd �resync' s zRobustUnpacker.resyncc C s$ | j r| j�|� n | j�|� d S rf )r] r\ r r^ r� )rb r� rc rc rd r� + s zRobustUnpacker.feedc C s | S rf rc rr rc rc rd �__iter__1 s zRobustUnpacker.__iter__c C s� | j r�d�| j�}| j r�|s t�t|| j�s:|dd � }qtjtd�| _ | j � |� zt | j �}W n tj tfyz Y n0 | � |�r�d| _ |S |dd � }qn t | j �S d S )Nr� r rY F)r] r� r\ r� rT rZ rA r� r7 r^ r� r� r� r[ )rb r� r� rc rc rd �__next__4 s& zRobustUnpacker.__next__) rw r� r� r� re r_ r� r` ra r$ rc rc r rd rU s rU c @ sZ e Zd Zdd� Zddd�Zd d � Zd d � Zd d� Zdd� Zddd�Z dd� Z ddd�Z dS )�ArchiveCheckerc C s d| _ t� | _d S �NF)� error_foundr� �possibly_supersededrr rc rc rd re P s zArchiveChecker.__init__FNr r� c C sP t �d� |du o t|||f� | _|| _|| _| �� | jsLt �d� dS | � |�| _ |rd| � � t j | jvr�t �d� d| _| �� | _nnz"t j|t jjf| j d�\| _} W nJ ty� } z2t �d| � d| _| jt j = | �� | _W Y d} ~ n d} ~ 0 0 | j|||||d � | �� | j| d � | j�r6t �d � n t �d � | j�pN| j S ) a� Perform a set of checks on 'repository' :param repair: enable repair mode, write updated or corrected data into repository :param archive: only check this archive :param first/last/sort_by: only check this number of first/last archives ordered by sort_by :param glob: only check archives matching this glob :param verify_data: integrity verification of data referenced by archives :param save_space: Repository.commit(save_space) z%Starting archive consistency check...NzJRepository contains no apparent data at all, cannot continue check/repair.Fz Repository manifest not found!T)r� z$Repository manifest is corrupted: %s)rS �first�last�sort_by�glob)� save_spacez3Archive consistency check complete, problems found.z6Archive consistency check complete, no problems found.)r� rM rS � check_all�repairr� � init_chunksr� r� �make_keyr� � verify_datar � MANIFEST_IDrd �rebuild_manifestr7 rI Z OperationZCHECK�IntegrityErrorBase�rebuild_refcounts�orphan_chunks_checkr� ) rb r� rl rS rf rg rh ri ro rj r� �excrc rc rd �checkT s: " zArchiveChecker.checkc C s` t t| j�d d�| _d}| jjt|d�}|s0q\|d }tdddd�}|D ]}|| j|< qJqdS )z8Fetch a list of all object keys from repository g�������?)ZusableN��limit�markerr r ��refcountrg rZ )r r r� r� �listrS r )rb ry �resultZ init_entryr� rc rc rd rm s zArchiveChecker.init_chunksc C s| d}| j �� D ]J\}}|d7 }|dkr* qZ|�|�}zt||�W S tyV Y q0 q|dkrhd}nd| }t|��d S )Nr r i� z*make_key: repository has no chunks at all!z4make_key: failed to create the key (tried %d chunks))r� � iteritemsr� r r r )rb r� ZattemptZchunkidr� �cdatar� rc rc rd rn � s zArchiveChecker.make_keyc C s� t �d� t| j�}d}d}g }t|dddd�}d }| jjd|d�}|sL�q�|t|�7 }|d }| j�|�}tt |��} | r4|� � | � d �} z t |�} W n~ t jtf�y } z^d | _|d 7 }t �d t| �| � t| t�r�|�| � | �rtt | ��}| j�|�}W Y d } ~ qxd } ~ 0 0 | tjk�r*d n| } z| j�| | � W qx t�y� } z6d | _|d 7 }t �d t| �|� |�| � W Y d }~qxd }~0 0 qxq4|�� ||k�r�t �d� t �d||� |�r�| j�rbt �d� |D ]�}z2| j� |�} |tjk�r�d n|} | j�| | � W n8 t�yJ | j|= | j� |� t � dt|�� Y n0 t �dt|�� �q�n&t �d� |D ]}t � dt|�� �qp|�r�t jnt j}|d||� d S )Nz5Starting cryptographic data integrity verification...r zVerifying data %6.2f%%�{�G�z�?zcheck.verify_data�r� r� �stepr� �d rw r Tr z chunk %s: %sz chunk %s, integrity error: %szGRepo/Chunks index object count vs. segment files object count mismatch.z:Repo/Chunks index: %d objects != segment files: %d objectszmFound defect chunks. They will be deleted now, so affected files can get repaired now and maybe healed later.zchunk %s deleted.z0chunk %s not deleted, did not consistently fail.z}Found defect chunks. With --repair, they would get deleted, so affected files could get repaired then and maybe healed later.zchunk %s is defect.z`Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.) r� rM r r� r; r� �scanr� r| �reversedr� r� r� rR r� rr rd r� r8 rk r r rp r� r� r� rl r� r� r� �debug)rb Zchunks_count_indexZchunks_count_segments�errorsZ defect_chunksr� ry Z chunk_idsZchunk_data_iterZchunk_ids_revdr� Zencrypted_datar~ Z _chunk_id�integrity_errorZ defect_chunk�logrc rc rd ro � s� � "$ � �zArchiveChecker.verify_datac s6 t dd� tD ��� � fdd�}t�d� t| j| j�}dd� tD �}tt | j �dd d d �}| j � � D �]�\}}|� � | j� |�}z| j�||�}W nB ty� } z*t�d | � d | _W Y d} ~ qdW Y d} ~ n d} ~ 0 0 t||�s�qdd|vsdd|vr�qdzt�|�} W n tj�y Y qdY n0 || �rdz| jj|dd�\} } }W nd t�y� } zJ| � dd��dd�} t�d| | � t�d� d | _W Y d} ~ qdW Y d} ~ n d} ~ 0 0 t| d�} | j} t�d| � | |jv �rd}d | |f }||jv�r�q�|d7 }�q�t� d | |� |} || j f|j| < qd|� � t�d � |S ) z�Rebuild the manifest object if it is missing Iterates through all objects in the repository looking for archive metadata blocks. c s s | ]}|� � V qd S rf �rW �r� r� rc rc rd rR � r� z2ArchiveChecker.rebuild_manifest.<locals>.<genexpr>c s t | t�sdS t| �}� �|�S rc )rk r� r� �issubset��obj�keys�Zrequired_archive_keysrc rd � valid_archive� s z6ArchiveChecker.rebuild_manifest.<locals>.valid_archivez9Rebuilding missing manifest, this might take some time...c S s g | ]}t �|�� ��qS rc rV rX rc rc rd r� r� z3ArchiveChecker.rebuild_manifest.<locals>.<listcomp>zRebuilding manifest %6.2f%%r� zcheck.rebuild_manifestr� z Skipping corrupted chunk: %sTNs cmdlines �versionFrN s names <unknown>�asciird �3Archive TAM authentication issue for archive %s: %szMThis archive will *not* be added to the rebuilt manifest! It will be deleted.r� zFound archive %sr z%s.%dz(Duplicate archive name %s, storing as %szManifest rebuild complete.) � frozensetZREQUIRED_ARCHIVE_KEYSr� rM r r� r� Z ARCHIVE_KEYSr; r r� r~ r� r� r� rr r� rd rT rA Zunpackbr� rP r �decoderI r� rG r� r� r� )rb r� r7 Zarchive_keys_serializedr� r� r� r r� ru rS �verifiedr� r� rL �new_namerc r� rd rq � sb � " " z ArchiveChecker.rebuild_manifestc s� �j �tjd� �fdd��� �fdd�}d+�fdd� � � ��fdd �}��fd d �}|du r�|�d �}t|||f�r�jjj||||d �} |r�| s�t � d|� |r�t | �|k r�t � d|t | �� |r�t | �|k r�t � d|t | �� n�jjj|d�} n<z�jj| g} W n( t �y2 t � d|� d�_Y dS 0 t | �} t| dddd�} t�j�����t| �D �]^\} } | �| � t �d| j� d| d � d| � d �� | j}|�j v�r�t � d t|�� d�_�jj| j= �qb�|� �j�|�}z�j�||�}W nX t�yP } z>t � d t|�|� d�_�jj| j= W Y d}~�qbW Y d}~n d}~0 0 z�jj |d d �\}}}W n` t�y� } zFt � d!| j|� t � d"� d�_�jj| j= W Y d}~�qbW Y d}~n d}~0 0 t |d#�}|j dk�r�t d$��d%d&� |j D �|_ t!�j�}||_"||�D ]&}d'|v �r0|| j|� |�#|� �q|j$dd(� |j%D ]}�|� �qP|j |_%�jj&|�'� d)|d*�}�j�(|�}�j�)|�}� |t |�t |�|� || j*f�jj| j< �qb| �+� W d � n1 �s�0 Y dS ),z�Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected Nc s, � j �| tddd��jdkr(� j�| � d S rV )r� r� r r{ re r� )r� rr rc rd �mark_as_possibly_supersededH s zEArchiveChecker.rebuild_refcounts.<locals>.mark_as_possibly_supersededc s2 �j �| �}�j �| �}� |t| �t|�|� |S rf )r� r" �encryptr )r r� r )� add_referencerb rc rd � add_callbackL s z6ArchiveChecker.rebuild_refcounts.<locals>.add_callbackc sP z� j �| � W n: tyJ td||d�� j | < � jrF� j�| |� Y n0 d S )Nr rz )r� Zincrefr� r rl r� Zput)r� rg rZ r rr rc rd r� R s z7ArchiveChecker.rebuild_refcounts.<locals>.add_referencec sr �fdd�}d}g }d}d|v }|j }|r0|jn|}|rlt|�t|�krlt�| � d|j� d�� |`d}|}t||�D �]n\} } | \} } } | �j v�r|| | kr�t�d�| |j||| t | ��� d �_ }|| �\} } } }� | | | |� n�t� d �| |j||| t | ��� | \} } } | �j v �r0� | | | � nJt�d �| |j||| t | ��� d �_ }|| �\} } } }� | | | |� nR| | k�r�� | | | � n:t� d �| |j||| t | ��� � | | | � �| d � |� | | | g� || 7 }qv|�r�|�s�|j |_|�r(||k�r(t� | � d|j� d �� |`||_ d|v �rn|j }|jdd d�}||k�rnt�d�| |j||�� dS )a Verifies that all file chunks are present. Missing file chunks will be replaced with new chunks of the same length containing all zeros. If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one. c s@ t d t| d�}t|� jj�\}}� j�|�}t|�}|| ||fS )N)r rg )r r r r� r" r� r )rg r r� r� r rZ rr rc rd �replacement_chunka s zWArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunks.<locals>.replacement_chunkr Fr� r� z*: Invalid chunks_healthy metadata removed!z^{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). Replacing with all-zero chunk.Tz|{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). It has an all-zero replacement chunk already.zm{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). Generating new replacement chunk.zE{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).z,: Completely healed previously damaged file!rg )� compressedr z<{}: {}: size inconsistency detected: size {}, chunks size {}N)r� r� r r� r� r� r� r� rq r8 rd rM r rg r )� archive_namer� r� �offsetZ chunk_listZchunks_replacedZhas_chunks_healthyZchunks_currentr� Z chunk_currentZ chunk_healthyr� rg rZ r r� r� )r� r� rb rc rd �verify_file_chunks[ sp � � � � �z<ArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunksc 3 s� t dd� �jjD ���t dd� tD ���tdd� �jj�}d� � �fdd�}�fd d �}d d � ����fd d�}d}t| j|�D �]B\}}t|�}|d r�|D ]}|d||� |d7 }q�q�|dkr�|�� t |�� |��D ]�\}} zX�j � || �} |� | � |D ]6} || �\} } | �r$t| d�V n|d| ||� �q W n~ t�yp } z |t|�||� W Y d}~nRd}~0 tj�y� |d||� |�� Y n" t�y� |d||� � Y n0 |d7 }q�q�dS )z�Iterates through all archive items Missing item chunks will be skipped and the msgpack stream will be restarted c s s | ]}|� � V qd S rf r� r� rc rc rd rR � r� zLArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<genexpr>c s s | ]}|� � V qd S rf r� r� rc rc rd rR � r� c S s t | t�od| v S )Ns path)rk r7 rk rc rc rd rl � r� zKArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<lambda>r c s" � d t | �jv�kr � d7 � � S )N� r )r� r� )r� )�_staterb rc rd �missing_chunk_detector� s zYArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.missing_chunk_detectorc s, t |�}| d||f 7 } d� _t�| � d S )Nz [chunk: %06d_%s]T)r8 rd r� r� )r� r� Zchunk_nor� rr rc rd �report� s zIArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.reportc S s d� dd� | D ��S )Nz, c s s, | ]$}t |t�r |jd d�nt|�V qdS )rd )r� N)rk r r� r� )r� �krc rc rd rR � r� zdArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safe.<locals>.<genexpr>)r� )r� rc rc rd �list_keys_safe� s zQArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safec sb t | t�sdS | �dd � t| �}��|�s@dd��| � fS |�� �s^dd�|� � fS dS )N)Fznot a dictionarys aclFzmissing required keys: zinvalid keys: )Tr� )rk r7 r� r� r� r� )rZ r� �required_item_keysrc rd � valid_item� s zMArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.valid_itemr� zitem metadata chunk missingr r� zDDid not get expected metadata dict when unpacking item metadata (%s)NzCUnpacker crashed while unpacking item metadata, trying to resync...z5Exception while decrypting or unpacking item metadata)r� r7 rZ ZREQUIRED_ITEM_KEYSrU r r� r| r_ r� r� r� r� r� rH r r� rA r� rR )rS r� r� r� r� rL �stater� r� r r� r� Zvalid�reasonr� )r� rb )r� rZ r� r� rd �robust_iterator� sL � $ z9ArchiveChecker.rebuild_refcounts.<locals>.robust_iterator�,)rh ri rf rg z.--glob-archives %s does not match any archivesz+--first %d archives: only found %d archivesz*--last %d archives: only found %d archives)rh zArchive '%s' not found.TzChecking archives %3.1f%%g�������?zcheck.rebuild_refcountsr� zAnalyzing archive z (r r� �)z%Archive metadata block %s is missing!z*Archive metadata block %s is corrupted: %sFrN r� zEThis archive will be *removed* from the manifest! It will be deleted.r� rO c S s g | ] }t |��qS rc rV rW rc rc rd r� ( r� z4ArchiveChecker.rebuild_refcounts.<locals>.<listcomp>r� rt ru )rw �salt)N),r� r� r rp r� rS r7 rG r| r� r� r r� r� rd r; rQ r� r� r� rM r� rx r8 r� r� r� r rP rI rQ rR rY r� r r� r� r� rz r{ r" r� r[ r� )rb rS rf rg rh ri r� r� r� Z archive_infosZ num_archivesr� rL rM Z archive_idr r� r� r� r� rF r� Zprevious_item_idZnew_archive_idrc )r� r� r� rb rd rs @ s� LH � $ $ $ z ArchiveChecker.rebuild_refcountsc C s� | j r�dd� | j�� D �}|| j }|rBt�t|�� d�� d| _| jr�|r�t� dt|�t| j�f � |D ]}| j � |� qlt� d� n t� d� d S )Nc S s h | ]\}}|j d kr|�qS )r )r{ )r� r� r� rc rc rd � <setcomp>< r� z5ArchiveChecker.orphan_chunks_check.<locals>.<setcomp>z orphaned objects found!Tz1Deleting %d orphaned and %d superseded objects...z.Finished deleting orphaned/superseded objects.z<Orphaned objects check skipped (needs all archives checked).) rk r� r~ re r� r� r rd rl rM r� r� )rb ZunusedZorphanedr� rc rc rd rt : s � z"ArchiveChecker.orphan_chunks_checkc C s8 | j r4t�d� | j�� t�d� | jjd|d� d S )NzWriting Manifest.zCommitting repo.F)rx rj )rl r� rM r7 r r� r{ )rb rj rc rc rd r� J s zArchiveChecker.finish)FNr r r� NFF)Nr r r� N)F) rw r� r� re rv rm rn ro rq rs rt r� rc rc rc rd rb N s � +QL {rb c @ s� e Zd ZG dd� de�Zedd� �Zd!dd �Zd"d d �Zd d � Z dd� Z dd� Z dd� Z dd� Z d#dd�Zdd� Zd$dd �Zd d � Zd d � ZdS )%�ArchiveRecreaterc @ s e Zd Zddd�ZdS )z ArchiveRecreater.InterruptedNc C s |pi | _ d S rf )rT )rb rT rc rc rd re T s z%ArchiveRecreater.Interrupted.__init__)N)rw r� r� re rc rc rc rd � InterruptedS s r� c C s | � d�S )N� .recreate)�endswith)r� rc rc rd �is_temporary_archiveW s z%ArchiveRecreater.is_temporary_archiveFNr3 c C s� || _ || _|| _|| _|| _|| _|p*g | _|| _| d u| _| jrPt � d| � | pVt | _ | | _ | | _| pptd�| _t� | _|| _| | _|| _|| _|p�dd� | _| r�d n|| _d S )NzRechunking archives to %sZnonec W s d S rf rc r rc rc rd rl w r� z+ArchiveRecreater.__init__.<locals>.<lambda>)r� r� r7 r r� �exclude_caches�exclude_if_present�keep_exclude_tagsr r� r� r� r � recompress�always_recompressr � compressionr� � seen_chunksr| r� rn rJ r r; )rb r� r7 r� r r� r� r� r� r r� r� r� r� rn rJ r r| r; rc rc rd re [ s, zArchiveRecreater.__init__c C s� | � |�}| �||�}| js"| jr,| �|� | j�� rV| jsV|jsV|d u rV|d u rVdS | � ||� |d u }| j ||||d� dS )NF)�replace_originalT) � open_archive� create_targetr� r� �matcher_add_tagged_dirsr� �emptyr� �recreate_rechunkify� process_itemsrq )rb r� r9 � target_namerS �targetr� rc rc rd �recreatez s& ����� zArchiveRecreater.recreatec s | j }|�� � � ri nd }� fdd�}|�� D ]�}|�|j�st| �d|j� ||�r0|�d�|�d�d f||j< q0� r�t|j�r�|�d�|v r�||j \}}} | d u r�||_ |d ur�||_ d d |jf||j < |` n| |_ | j r�| �d|j� q0| � |||� q0| j�r|jjdd � d S ) Nc s" � o t | j�o | �dd�o d| vS r� r� rk �Ztarget_is_subsetrc rd �item_is_hardlink_master� s � ��z?ArchiveRecreater.process_items.<locals>.item_is_hardlink_master�xr� r� r� �-T)r� )r� r� rm r� r� r r� r" r� r� r� r� r� � process_itemrJ rn r� ) rb rS r� r� r� r� r� r� r� Z new_sourcerc r� rd r� � s0 z ArchiveRecreater.process_itemsc C sd t |j�}d|v rB| �||j� d }| �|||� |j jd7 _|j||jd� | �||j� d S )Nr� r r )r1 r� r r� �process_chunksrn r\ rp )rb rS r� r� r" rc rc rd r� � s z ArchiveRecreater.process_itemc C sr | j s4|js4|jD ]\}}}| j�||j� q|jS | �||t|j��}t| j |�}|� || j|j| j ||� d S rf ) r� r� r� r r rn � iter_chunksr| r r r rJ ) rb rS r� r� r� rg rZ �chunk_iteratorr rc rc rd r� � s z ArchiveRecreater.process_chunksc C s� t || jj�\}}|| jv r,| j�||j�S | j}| jr�| js�|| jj v r�| jj d | j � |�dd�}t �|�j| jj�|�jkr�d}| jj|||j|dd�}| jj jdd� | j�|j� |S )NF)� decompress)� overwriter r )r r� r" r� r r rn r� r� r� r� r� r� r Zdetectr� Z compressorZdecider! r# r� rx )rb r� r r� r� r� Z old_chunkr rc rc rd r � s z ArchiveRecreater.chunk_processorc c sZ |j �dd� |D ��}|jr8t|�}|j�|�E d H n |D ]}t|t|�td�V q<d S )Nc S s g | ]\}}}|�qS rc rc )r� r� r� rc rc rd r� � r� z0ArchiveRecreater.iter_chunks.<locals>.<listcomp>)rg r ) rD r� r� r# r r r r r )rb rS r� r� r� r� r rc rc rd r� � s z ArchiveRecreater.iter_chunksTc C s� | j r d S |d u r |j�dd�}| jr,|j}| jd u r`|jj|j�d�pN|jj|jjtj d�}n|jjtj d�}|j || j|d� |r�|j t � | j d� |�|j� | jr�||_t� |_ttt|�tt|j�t| j�t� d S )Nr9 r� r\ )r� r\ rY �recreate_cmdline)rY r� )r9 r| r} )rJ )r� rT r� rn rA r| r� rY r� rh rq r� rU rJ r� r� rC r� r< ZDASHESr� r )rb rS r� r9 r� Z_startr} rc rc rd rq � s: � � �zArchiveRecreater.savec s6 ���fdd�}�j � g �g �i }�jrd|jdd� d�D ]*}t�|j�r8d|vr8d|v r8d||j< q8|j� fd d�d�D ]�}�jr�|j|v r�|||j< tj� |j�\}}|�j v r�|||� qx�jrx|t krxt�|j�rxd|v r�|n||j }t ||�}|� tt��tkrx|||� qx� ��tj� � ��tj� dS ) zLAdd excludes to the matcher created by exclude_cache and exclude_if_present.c sH � j r2��t|jdd�� ��t| d dd�� n��t| dd�� d S )NF)Z recurse_dirr� )r� r rE r� rF )�dirZtag_item)rb � tag_files� tagged_dirsrc rd �exclude s z9ArchiveRecreater.matcher_add_tagged_dirs.<locals>.excludec S s t j�| j�tkS rf )r� r� �basename�CACHE_TAG_NAMErk rc rc rd rl r� z:ArchiveRecreater.matcher_add_tagged_dirs.<locals>.<lambda>r� r� r� Nc s t j�| j�tkp � �| j�S rf )r� r� r� r� r� rk r� rc rd rl $ r� )r� r� rm r� r� r� r� r� r� r� r� r� r% r� r ZCACHE_TAG_CONTENTSr� rG ZIncludeZExcludeNoRecurse) rb rS r� Zcachedir_mastersr� r� Ztag_fileZ content_itemr� rc )r� rb r� r� rd r� s0 � z(ArchiveRecreater.matcher_add_tagged_dirsc C s� |p |j d }| �|�}|j�d�}|dur4t|�nd}|j}| joJ||k|_|jrft� d|p`d|� t | j | j |j |j| j|jd�j|_t|j| j jdd��|_|S ) zCreate target archive.r� r Nz Rechunking archive from %s to %sz (unknown))r r� rp rs r; r Fr� )r� �create_target_archiverT r� r$ r r r� r� r� r r r� rp rs r; r r r� r )rb rS r� r� Zsrc_cpZdst_cprc rc rd r� 2 s � z ArchiveRecreater.create_targetc C s, t | j| j| j|d| j| j| j| jd� }|S )NT)rE rJ r r r; )r/ r� r� r7 rJ r r r; )rb r� r� rc rc rd r� D s �z&ArchiveRecreater.create_target_archivec K s" t | j| j| j|fd| ji|��S )Nr )r/ r� r� r7 r )rb r� �kwargsrc rc rd r� J s z ArchiveRecreater.open_archive) FNFNNFFFFFNNr3 )NN)NT)N)rw r� r� rR r� r� r� re r� r� r� r� r r� rq r� r� r� r� rc rc rc rd r� R s& � % '* r� )�r� r� Zsocketr� r� r� � collectionsr � contextlibr Zdatetimer r � functoolsr Zgetpassr �ior � itertoolsr r �shutilr Z platformflagsr r r r r� r r� r r r r r r Z crypto.keyr r �compressr r Z constantsZcrypto.low_levelr rr Z hashindexr r r Zhelpersr r" r# r$ r% r& r' �platformr( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC ZlrucacherD �patternsrE rF rG r� rH rI rJ rK rL rM rN rO rP ZremoterQ r� rR rS r� r� rU r� rR r� r� r� r� r� r� r� r� r� r r. r/ r� r r r r r> rT rU rb r� rc rc rc rd �<module> s� w D1 L=M 89 2
Simpan