asda?‰PNG  IHDR ? f ??C1 sRGB ??é gAMA ±? üa pHYs ? ??o¨d GIDATx^íüL”÷e÷Y?a?("Bh?_ò???￠§?q5k?*:t0A-o??￥]VkJ￠M??f?±8\k2íll￡1]q?ù???T 3 updC@s*ddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z e Z e dZ ddl mZmZdd lmZmZmZdd lmZdd lmZdd lmZdd lmZmZddlmZmZmZmZddlmZddlmZddlm Z m!Z!ddlm"Z"ddlm#Z#m$Z$ddlm%Z%m&Z&ddlm'Z'ddlm(Z(ddl)m*Z*m+Z+ddl,m-Z-ddl.m/Z/m0Z0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9eddZ:Gd d!d!Z;d"d#Zd(d)Z?d*d+Z@Gd,d-d-ZAGd.d/d/ZBGd0d1d1ZCGd2d3d3eCZDGd4d5d5eCZEdS)7N) unhexlify) namedtuple) perf_counter) create_loggerzborg.debug.files_cache) CACHE_READMEFILES_CACHE_MODE_DISABLED) ChunkIndexChunkIndexEntryCacheSynchronizer)Location)Error)Manifest) get_cache_dirget_security_dir) int_to_bigint bigint_to_int bin_to_hexparse_stringified_list)format_file_size)safe_ns)yeshostname_is_unique)remove_surrogates)ProgressIndicatorPercentProgressIndicatorMessage)set_ec EXIT_WARNING) safe_unlink)msgpack) ArchiveItemChunkListEntry) PlaintextKey)IntegrityCheckedFileDetachedIntegrityCheckedFileFileIntegrityError)Lock)SaveFile)cache_if_remote)LIST_SCAN_LIMITFileCacheEntryzage inode size cmtime chunk_idsc@seZdZdZddZedddZddZd d Zd d Z dd dZ dddZ dddZ ddddddZ dddZddZdS) SecurityManageras Tracks repositories. Ensures that nothing bad happens (repository swaps, replay attacks, unknown repositories etc.). This is complicated by the Cache being initially used for this, while only some commands actually use the Cache, which meant that other commands did not perform these checks. Further complications were created by the Cache being a cache, so it could be legitimately deleted, which is annoying because Borg didn't recognize repositories after that. Therefore a second location, the security database (see get_security_dir), was introduced which stores this information. However, this means that the code has to deal with a cache existing but no security DB entry, or inconsistencies between the security DB and the cache which have to be reconciled, and also with no cache existing but a security DB entry. cCsV||_t|j|_t||_tjj|jd|_tjj|jd|_ tjj|jd|_ dS)Nzkey-typelocationzmanifest-timestamp) repositoryrid_strdir cache_dirospathjoin key_type_file location_filemanifest_ts_file)selfr-r8/usr/lib64/python3.6/cache.py__init__>s   zSecurityManager.__init__NcCs(|p t|j}tjj|r$tj|dS)z:destroy the security dir for ``repository`` or at ``path``N)rr.r1r2existsshutilrmtree)r-r2r8r8r9destroyFs zSecurityManager.destroycCstdd|j|j|jfDS)Ncss|]}tjj|VqdS)N)r1r2r;).0fr8r8r9 Nsz(SecurityManager.known..)allr4r5r6)r7r8r8r9knownMszSecurityManager.knowncCsn|js dSy.t|jd}|j}|t|jkSQRXWn.tk rh}ztjd|WYdd}~XnXdS)NFrz&Could not read/parse key type file: %s) rCopenr4readstrTYPEOSErrorloggerwarning)r7keyfdtypeexcr8r8r9 key_matchesQszSecurityManager.key_matchescCstjd|jj|j|jjj}tjd|tjdt|jtjd|j t |j }|j |WdQRXt |j }|j t|jWdQRXt |j}|j |j WdQRXdS)Nz#security: saving state for %s to %szsecurity: current location %szsecurity: key type %szsecurity: manifest timestamp %s)rJdebugr-r.r/ _locationcanonical_pathrGrH timestampr'r5writer4r6)r7manifestrLZcurrent_locationrMr8r8r9save[s     zSecurityManager.savecCs,y.t|j}|j}WdQRXtjd|WnVtk rTtjd|jd}Yn2tk r}ztjd|d}WYdd}~XnX|r|jr||jkr|j}tjd||j j j }|o||kr(dj ||d}t |ddd d d stjtjd t|j}|j|WdQRX|r(|jdS) Nz#security: read previous location %rz-security: previous location file %s not foundz)Could not read previous location file: %sz.security: using previous_location of cache: %rzDWarning: The repository at location {} was previously located at {} zDo you want to continue? [yN] z Aborting.zInvalid answer, aborting.FZ BORG_RELOCATED_REPO_ACCESS_IS_OK) false_msg invalid_msgretryenv_var_overridezrCrPrWrcrirlrrrpror8r8r8r9r+*s     r+cCst|}|j||j|ddS)N)rn)r+rrrL)r-rVrnZsmr8r8r9rrsrrcCs|jj}t|}t|}|j|jkr|j|jkr|j|jkr|j|jkr|jr|jr|jjdr|jjdr|jdd|jddkr|S|SdS)Nz/~/z/./) rRrSr protouserhostZportr2 startswith)Zcache_locationr-Z repo_locationZrlZclr8r8r9 recanonicalize_relative_locations 0 4r~cCs|ptjjt|jS)N)r1r2r3rr.)r-r2r8r8r9r0sr0cCstjjdd}|rd|SdS)NZBORG_FILES_CACHE_SUFFIXrdzfiles.files)r1environget)suffixr8r8r9files_cache_namesrcCsddtj|DdS)NcSs"g|]}|dks|jdr|qS)rzfiles.)r})r?fnr8r8r9 sz-discover_files_cache_name..r)r1listdir)r2r8r8r9discover_files_cache_namesrc@s`eZdZdddZddZddZdd Zd d Zd d ZddZ dddZ ddZ ddZ dS)rqNcCs4||_t|||_tjj|jd|_d|_||_dS)Nconfig)r-r0r2r1r3 config_pathlockrn)r7r-r2rnr8r8r9r:s  zCacheConfig.__init__cCs |j|S)N)rE)r7r8r8r9 __enter__szCacheConfig.__enter__cCs |jdS)N)close)r7exc_typeexc_valexc_tbr8r8r9__exit__szCacheConfig.__exit__cCstjj|jS)N)r1r2r;r)r7r8r8r9r;szCacheConfig.existsc Cs|j sttjdd}|jd|jddd|jdd|jj|jddd|jd|jdddt|j }|j |WdQRXdS) N) interpolationcacheversion1r-rVrd integrity) r;AssertionError configparser ConfigParser add_sectionsetr-r.r'rrU)r7rrMr8r8r9creates    zCacheConfig.createcCs2ttjj|jdd|jtdj|_|jdS)NrT) exclusiveZtimeoutZkill_stale_locks) r&r1r2r3rnracquirerload)r7r8r8r9rEszCacheConfig.opencCsttjdd|_t|j}|jj|WdQRX|j|j|jjdd|_t |jjdd|_ |jjdddd|_ |jjdddd|_ t t|jjddd d|_t t|jjdd d d|_y@t|jjd |_|jjdd|jjdkri|_tjd Wn(tjk r(tjd i|_YnX|jjdddd}|rRt||j|_nd|_|jj dd|jjjdS)N)rrr-rVrT)Zfallbackrjignored_featuresrdmandatory_featuresrzHCache integrity data not available: old Borg version modified the cache.zTCache integrity: No integrity data found (files, chunks). Cache is from old version.r])rr_configrErZ read_file_check_upgraderidr manifest_idrTrjrrrrdictitemsrpoprJrKNoSectionErrorrQr~r-r]rRrS)r7rMr]r8r8r9rs.    zCacheConfig.loadc Cs|r|jjdd|j|jjdd|j|jjdddj|j|jjdddj|j|jjdsp|jjdx&|j j D]\}}|jjd||q|W|jjdd|j|r|jjddt |j t |j}|jj|WdQRXdS) NrrVrTr,rrrj)rrr.rTr3rrZ has_sectionrrrrGrHr'rrU)r7rVrLfileintegrity_datarMr8r8r9rW3s   zCacheConfig.savecCs|jdk r|jjd|_dS)N)rrelease)r7r8r8r9rCs  zCacheConfig.closec Csjy8|jjdd}d}||kr6|jtd|||fWn,tjk rd|jtd|dYnXdS)Nrrrz0%s has unexpected cache version %d (wanted: %d).z#%s does not look like a Borg cache.)rZgetintr Exceptionrr)r7rZ cache_versionZwanted_versionr8r8r9rHszCacheConfig._check_upgrade)NN)NN) rtrurvr:rrr;rrErrWrrr8r8r8r9rqs   rqc @seZdZdZGdddeZGdddeZGdddeZGdd d eZGd d d eZ e dd dZ e dddZ d ddddd de df ddZd S)r_zClient Side cache c@seZdZdZdS)zCache.RepositoryIDNotUniquez`Cache is newer than repository - do you have multiple, independently updated repos with same ID?N)rtrurvrwr8r8r8r9rgXsrgc@seZdZdZdS)zCache.RepositoryReplayzCache, or information obtained from the security directory is newer than repository - this is either an attack or unsafe (multiple repos with same ID)N)rtrurvrwr8r8r8r9rh[srhc@seZdZdZdS)zCache.CacheInitAbortedErrorzCache initialization abortedN)rtrurvrwr8r8r8r9rs^srsc@seZdZdZdS)zCache.RepositoryAccessAbortedzRepository access abortedN)rtrurvrwr8r8r8r9r`asr`c@seZdZdZdS)zCache.EncryptionMethodMismatchzLRepository encryption method changed since last access, refusing to continueN)rtrurvrwr8r8r8r9rkdsrkNcCs(t||}ttjj|dddjdS)NrT)r)r0r&r1r2r3 break_lock)r-r2r8r8r9rgs zCache.break_lockcCsH|ptjjt|j}tjj|d}tjj|rDtj|tj|dS)z3destroy the cache for ``repository`` or at ``path``rN) r1r2r3rr.r;remover<r=)r-r2rr8r8r9r>ls   z Cache.destroyTFc  s| rdkrdn"| r6dkr6djttdf dd} fdd}| sn| St}|jr||jjk}WdQRX|rtjd| Stjd |S) Ndirdc std S)N) r-rLrVr2syncrmprogressrn cache_mode) LocalCacher8) rrLrnrVr2rr-rrmr8r9local~s zCache.__new__..localcstdS)N)r-rLrVrn) AdHocCacher8)rLrnrVr-r8r9adhocszCache.__new__..adhocz%Cache: choosing local cache (in sync)zKCache: choosing ad-hoc cache (local cache does not exist or is not in sync))r3rrqr;rrrJrQ)clsr-rLrVr2rZdo_filesrmrrnZpermit_adhoc_cacherZ ignore_inoderrraZ cache_in_syncr8) rrLrnrVr2rr-rrmr9__new__us"    z Cache.__new__)N)N)rtrurvrwr rgrhrsr`rkrxrr>rrr8r8r8r9r_Us  r_c@sFeZdZdZddZedddddd d gZd d Zd dZddZ dS)CacheStatsMixinzAll archives: {0.total_size:>20s} {0.total_csize:>20s} {0.unique_csize:>20s} Unique chunks Total chunks Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}cCs|jj|jS)N) str_formatr^ format_tuple)r7r8r8r9__str__szCacheStatsMixin.__str__Summary total_size total_csizeZ unique_size unique_csizeZtotal_unique_chunksZ total_chunkscCs|j|jjj}|S)N)rchunksZ summarize_asdict)r7statsr8r8r9rszCacheStatsMixin.statscCs2|j}xdD]}t||||<qW|jf|S)Nrrr)rrr)rrr)r7rZfieldr8r8r9rs zCacheStatsMixin.format_tuplecCs |jdS)Nr)r)r7r8r8r9chunks_stored_sizesz"CacheStatsMixin.chunks_stored_sizeN) rtrurvrrrrrrrr8r8r8r9rsrc@seZdZdZdddddefddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd/d#d$Zd0d%d&Zd1d'd(Zd2d)d*Zd+d,Zd-d.ZdS)3rz0 Persistent, local (client-side) cache. NTFc Cs||_||_||_||_| |_d|_d|_t|||_t ||_ t |j|j||_ t jj|jsx|j j||||j|jyT|j j|||j d|js|j|j|r|jj|j jkr|j|jWn|jYnXdS)aO :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository :param lock_wait: timeout for lock acquisition (int [s] or None [wait forever]) :param sync: do :meth:`.sync` :param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison NF)ra)r-rLrVrrrT txn_activer0r2r+security_managerrqrar1r;rorrErrcheck_cache_compatibility wipe_cacheupdate_compatibilityrrrcommitr) r7r-rLrVr2rrmrrnrr8r8r9r:s2   zLocalCache.__init__cCs|S)Nr8)r7r8r8r9rszLocalCache.__enter__cCs |jdS)N)r)r7rrrr8r8r9rszLocalCache.__exit__cCstj|jttjj|jdd}|jtWdQRX|jjt jtjj|jdtjtjj|jdt tjj|jt ddWdQRXdS)z0Create a new empty cache at `self.path` ZREADMEwNrzchunks.archive.dT)binary) r1makedirsr2rEr3rUrrarr r'r)r7rMr8r8r9rs  zLocalCache.createc Csd|jjttjj|jdd|jjjdd}tj ||_ WdQRXd|j krXd|_ n|j dS)NrF)r2rUrr)rarr#r1r2r3rrr rFrrr _read_files)r7rMr8r8r9_do_opens  zLocalCache._do_opencCs2tjj|jstd|j|jj|jdS)Nz"%s Does not look like a Borg cache)r1r2isdirrrarErollback)r7r8r8r9rEs zLocalCache.opencCs|jdk r|jjd|_dS)N)rar)r7r8r8r9rs  zLocalCache.closec Csi|_d|_tjdtjdd}yttjj|jt d|j j j t d}t jdd}x|jd}|snP|j|yFILES-CACHE-KILL: removed all old entries with age >= TTL [%d]zCFILES-CACHE-KILL: removed all current entries with newest cmtime %dz7FILES-CACHE-SAVE: finished, %d remaining entries saved.zSaving chunks cacherzSaving cache configz txn.activeztxn.tmpFll)$rrrWrVrLrrrintr1rrrrrQr#r2r3rrr*runpackbrrcmtimepackrrarrrUrr<r=r)r7rZttlrMZ entry_countrrrr8r8r9r@sB         zLocalCache.commitcCstjjtjj|jdr.tjtjj|jdtjj|jd}tjj|rtjtjj|d|jtjtjj|d|jtjtjj|t||jtj|tjj|jdtjjtjj|jdrtjtjj|jdd|_ |j dS)z3Roll back partial and aborted transactions ztxn.tmpz txn.activerrFN) r1r2r;r3r<r=rrrrr)r7rr8r8r9ris zLocalCache.rollbackcsJtjjjdtd ddd d dfdd fddfdd fd d d fd d  fdd fdd fdd fddfdd  fdd}fdd}jjtjjfj t j j d&|tjj _|j_WdQRXdS)!aRe-synchronize chunks cache with repository. Maintains a directory with known backup archive indexes, so it only needs to fetch infos from repo and build a chunk index once per backup archive. If out of sync, missing archive indexes get added, outdated indexes get removed and a new master chunks index is built by merging all archive indexes. zchunks.archive.drrdcst|}tjj||}|S)N)rr1r2r3)rrZid_hexr2) archive_pathr8r9mkpathszLocalCache.sync..mkpathcs>jr4tj}tdd|Dtdd|DBStSdS)Ncss"|]}t|dkrt|VqdS)rN)rr)r?rr8r8r9rAsz;LocalCache.sync..cached_archives..css4|],}t|dkr|jdrt|ddVqdS)Hz.compactNr)rendswithr)r?rr8r8r9rAs)do_cacher1rr)Zfns)rr7r8r9cached_archivess  z(LocalCache.sync..cached_archivescstddjjjDS)Ncss|] }|jVqdS)N)r)r?infor8r8r9rAsz9LocalCache.sync..repo_archives..)rrVarchiveslistr8)r7r8r9 repo_archivessz&LocalCache.sync..repo_archivescsx|D] }|qWdS)Nr8)idsr)cleanup_cached_archiver8r9cleanup_outdateds z)LocalCache.sync..cleanup_outdatedTcsy$tj|tj|dWntk r8YnX|sBdSy,tj|ddtj|dddWntk rYnXdS)Nz .integrityz.compact)r)r1unlinkr\)rcleanup_compact)rr8r9rsz/LocalCache.sync..cleanup_cached_archivecs|j}g}trlxZ|D]L}j|}|r\||j|jd}|j|jksRtd|||<q|j|qWn|}xVt|j j |D]@\}}||jt|d}|||<||<d7t|7qWdS)z Archives created with AdHocCache will have csize=0 in all chunk list entries whose chunks were already in the repository. Scan *chunk_idx* for entries where csize=0 and fill in the correct information. )csizezChunk size mismatchrN) Zzero_csize_idsrrrrsizerappendzipr-get_many) chunk_idxZall_missing_idsZ fetch_idsid_Zalready_fetched_entryrr)chunks_fetched_size_indexdecrypted_repositoryfetched_bytes_for_csizefetched_chunks_for_csizer8r9fetch_missing_csizes"    z,LocalCache.sync..fetch_missing_csizec s|j|\}}|j|dt||jj|dd\}}}t|d}|jdkrTtdt|}xVt |j |j |j D]>\} \}}|j| dt||t|7d7|j |qrWj r̈|||dS)NrT)Zforce_tam_not_required)Z internal_dictz Unknown archive metadata version)raddrrLZunpack_and_verify_archiver rrr rrrrr) archive_idr rrrarchiveZverified_rZitem_id)rprocessed_item_metadata_bytesprocessed_item_metadata_chunksr7write_archive_indexr8r9fetch_and_build_idxs  " z,LocalCache.sync..fetch_and_build_idxcs|j7|dd}|dd}y0t|dt|dd}|j|WdQRXWntk rpt|YnXtj||dS)Nz.compact)rz.tmpT)r2rUfilename)Zcompactr$rrUrrr1r)rrrZfn_tmprM)"compact_chunks_archive_saved_spacerr8r9rs   z,LocalCache.sync..write_archive_indexcs|}tjd|yvy:t|ddd}tj|dd}WdQRX|dd|Stk rt|dd}tj|}WdQRXYnXWn>tk r}z"tjd|||tt dSd}~XnXtj d |||||S) Nz-Reading cached archive chunk index for %s ...z.compactF)r2rUT)Zpermit_compact)rz1Cached archive chunk index of %s is corrupted: %sz6Found non-compact index for %s, converting to compact.) rJrr$r rFr\r%errorrrrQ)r archive_nameZarchive_chunk_idx_pathrMarchive_chunk_idxr)rrrr8r9read_archive_indexs(    z+LocalCache.sync..read_archive_indexcsHi}x*jjjD]}|j|kr|j||j<qWt|t|ksDt|S)N)rVrrrnamerr) archive_idsZ archive_namesr)r7r8r9get_archive_ids_to_namess  z1LocalCache.sync..get_archive_ids_to_namesc stjd} }tjdt|t|t||t|||j||tt jtj}|r jsxdnt|}t t|dddd}|}x|j D]\}}|j t |gd jr*||kr ||}|dkr|j |||krtjd|t}||tjd |j|q|p6t|}tjd |||qW jsd||jtjd ttjd t  tjd ttjd|S)NzSynchronizing chunks cache...zIArchives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.g?z3%3.0f%% Syncing chunks cache. Processing archive %sz cache.sync)totalsteprbr)rz.Fetching and building archive index for %s ...z$Merging into master chunks index ...z!Fetching archive index for %s ...zlCache sync: had to fetch %s (%d chunks) because no archive had a csize set for them (due to --no-cache-sync)z0Cache sync: processed %s (%d chunks) of metadataz;Cache sync: compact chunks.archive.d storage saved %s byteszDone.)rJrrclearrr-r MAX_LOAD_FACTORrrrshowrrmergerrQr) rZ cached_idsrZmaster_index_capacityrZarchive_ids_to_namesrrr)rrrr rrr r rrrrrr7r8r9create_master_idxsT                z*LocalCache.sync..create_master_idxcspytjtjjjdWn YnXytjtjjjdWn YnXytjWn YnXdS)z?bring old cache dirs into the desired state (cleanup and adapt)zchunks.archivezchunks.archive.tmpN)r1rr2r3rr8)rr7r8r9legacy_cleanupQsz'LocalCache.sync..legacy_cleanup)Zdecrypted_cacheN)rd)T)r1r2r3r rVZcheck_repository_compatibilityrZ OperationZREADrr(r-rLrrr)r7r&r'r8)rrr rrrr rrr r rrrrrrr7rr9r{s2   ! &3zLocalCache.synccCs*tj}|jj|@rdS|jj|ks&dSdS)NFT)rSUPPORTED_REPO_FEATURESrarr)r7 my_featuresr8r8r9rms   z$LocalCache.check_cache_compatibilityc Cstjdtjj|jd}tjj|rRtjtjj|jdtjtjj|jdt |_ t tjj|jt ddWdQRXd|j _|j jjdddt|j _t|j _dS)Nz9Discarding incompatible cache and forcing a cache rebuildzchunks.archive.dT)rrdrrV)rJrKr1r2r3rr<r=rr rr'rrarrrrr)r7rr8r8r9rys    zLocalCache.wipe_cachecCs^|jj}tj}t}x|jD]\}}|j|q W|jjj|||jj j||@dS)N) rVZget_all_mandatory_featuresrr(rrupdaterarr)r7Zoperation_to_features_mapr)Z repo_featuresZ operationZfeaturesr8r8r9rs zLocalCache.update_compatibilityc Cs|js|jt|}|j||}|r8| r8|j||S|jj|}t|} |jj|||d|j j |d|| |j || | t ||| S)N)waitr) rrr seen_chunk chunk_increfrLencryptr-putrrr*r!) r7rchunkr overwriter+rrefcountrrr8r8r9 add_chunks    zLocalCache.add_chunkcCsJ|jj|tddd\}}}|dk rF|dk rF||krFtd|||f|S)Nrz@chunk has same id [%r], but different size (stored: %d new: %d)!)rrr r)r7rrr2Z stored_sizerr8r8r9r,s zLocalCache.seen_chunkcCs:|js|j|jj|\}}}|j||dt|||S)NF)rrrincrefr*r!)r7rrrcount_sizerr8r8r9r-s zLocalCache.chunk_increfcCsj|js|j|jj|\}}}|dkrT|j|=|jj||d|j| | dn|j| | ddS)Nr)r+TF)rrrdecrefr-deleter*)r7rrr+r5rrr8r8r9 chunk_decrefszLocalCache.chunk_decrefcCs4tj|jsdS|j}d|kr,tjddSd|krBtjddS|jj|}|sbtjd|dStt j |}d|kr|j |j krtjd |dSd |kr|j |jkrtjd |dSd |kot|j|jkrtjd|dSd|krt|j|jkrtjd|dSt j|j|jdd|j|<d |jfS)aj Check if we know the file that has this path_hash (know == it is in our files cache) and whether it is unchanged (the size/inode number/cmtime is same for stuff we check in this cache_mode). :param hashed_path: the file's path as we gave it to hash(hashed_path) :param path_hash: hash(hashed_path), to save some memory in the files cache :param st: the file's stat() result :return: known, ids (known is True if we have infos about this file in the cache, ids is the list of chunk ids IF the file has not changed, otherwise None). FNrzUNKNOWN: files cache disabledrDzUNKNOWN: rechunking enforcedz*UNKNOWN: no file metadata in cache for: %rsz(KNOWN-CHANGED: file size has changed: %rTrz0KNOWN-CHANGED: file inode number has changed: %rcz)KNOWN-CHANGED: file ctime has changed: %rmz)KNOWN-CHANGED: file mtime has changed: %rr)inoder)FN)FN)FN)FN)TN)TN)TN)TN)statS_ISREGst_moderrrQrrr*rrrst_sizer=st_inorr st_ctime_ns st_mtime_nsrr chunk_ids)r7 hashed_pathrstrrr8r8r9file_known_and_unchangeds8          z#LocalCache.file_known_and_unchangedc Cstj|jsdS|j}d|kr,tjddSd|krDd}t|j}nd|krZd}t|j}t d|j |j t ||d}t j||j|<t|jpd||_tjd |jd t|jd ||dS) Nrz*FILES-CACHE-NOUPDATE: files cache disabledr;Zctimer<mtimer)rr=rrrEz)FILES-CACHE-UPDATE: put %r [has %s] <- %rz [%d entries])rE)r>r?r@rrrQrrCrDr*rBrArrrrrerrrrE) r7rFrrGrrZ cmtime_typeZ cmtime_nsrr8r8r9 memorize_files$    zLocalCache.memorize_file)FT)N)N)T)rtrurvrwrr:rrrrrErrrrrrrrrr3r,r-r9rHrJr8r8r8r9rs0(  !)s    4rc@seZdZdZdZdddZddZd d ZdZd Z d d Z ddZ d ddZ d!ddZ d"ddZd#ddZddZddZddZdS)$ra Ad-hoc, non-persistent cache. Compared to the standard LocalCache the AdHocCache does not maintain accurate reference count, nor does it provide a files cache (which would require persistence). Chunks that were not added during the current AdHocCache lifetime won't have correct size/csize set (0 bytes) and will have an infinite reference count (MAX_VALUE). zAll archives: unknown unknown unknown Unique chunks Total chunks Chunk index: {0.total_unique_chunks:20d} unknownTNcCsB||_||_||_d|_t||_|jj|||dtjddS)NF)rnz1Note: --no-cache-sync is an experimental feature.) r-rLrV _txn_activer+rrrrJrK)r7r-rLrVrmrnr8r8r9r:s zAdHocCache.__init__cCs|S)Nr8)r7r8r8r9r"szAdHocCache.__enter__cCsdS)Nr8)r7rrrr8r8r9r%szAdHocCache.__exit__rcCstjddS)Nz$UNKNOWN: files cache not implementedF)FN)rrQ)r7rFrrGr8r8r9rH+s z#AdHocCache.file_known_and_unchangedcCsdS)Nr8)r7rFrrGrr8r8r9rJ/szAdHocCache.memorize_fileFc Cs| std|js|jt|}|j||}|rD|j|||dS|jj|}t|} |jj |||d|j j |d|| |j || | t ||| S)NuHAdHocCache does not permit overwrites — trying to use it for recreate?)r)r+r)rrK _begin_txnrr,r-rLr.r-r/rrr*r!) r7rr0rr1r+rr2rrr8r8r9r32s  zAdHocCache.add_chunkcCsN|js|j|jj|tddd}|jrH|rH|j rH|j|d|j|<|jS)Nr)r)rKrLrrr r2rr)r7rrrr8r8r9r,As zAdHocCache.seen_chunkcCsJ|js|j|jj|\}}}|p&|}|s0t|j||dt|||S)NF)rKrLrr4rr*r!)r7rrrr5r6rr8r8r9r-LszAdHocCache.chunk_increfcCsj|js|j|jj|\}}}|dkrT|j|=|jj||d|j| | dn|j| | ddS)Nr)r+TF)rKrLrr7r-r8r*)r7rrr+r5rrr8r8r9r9WszAdHocCache.chunk_decrefcCs&|js dS|jj|j|jd|_dS)NF)rKrrWrVrL)r7r8r8r9rbszAdHocCache.commitcCsd|_|`dS)NF)rKr)r7r8r8r9rhszAdHocCache.rollbackc Csd|_t|j}t|tjd}t||_t|ddd}t}d}d}xb|jj t |d}|d7}|shP|j t|d |d}t tj ddd }x|D]} ||j| <qWqJWt|j|kst|j|jj=t|pd } |jtjd || |t|d | dS)NTg?z!Downloading chunk list... %3.0f%%zcache.download_chunks)r rbrr)limitmarkerr)Zincrease)r2rrg{Gz?zBAdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s")rKrr-rr r#rrrrr)r$r Z MAX_VALUErrVZ MANIFEST_IDrrJrQr) r7Z num_chunksZcapacityrZt0Z num_requestsrNresultZ init_entryr Zdurationr8r8r9rLns2    zAdHocCache._begin_txn)TN)FT)N)N)T)rtrurvrwrr:rrrrrHrJr3r,r-r9rrrLr8r8r8r9rs    r)N)Frr1r<r>Zbinasciir collectionsrZtimerrJrrZ constantsrrZ hashindexr r r Zhelpersr r rrrrrrrrrrrrrrrrrrrr r!Z crypto.keyr"Zcrypto.file_integrityr#r$r%Zlockingr&platformr'Zremoter(r-r)r*r+rrr~r0rrrqr_rrrr8r8r8r9s`                  % eES