o )j6@sdZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZddlZddlZddlmZeZdZdZGd d d ZGd d d eeZd edefddZGdddZGdddZdS)zUnified file-level upload tracker. Merges hash cache and upload progress into a single .ms_upload_cache file with per-file status tracking, eliminating batch-granularity issues. N)Enum)Path)DictListOptionalTupleUnion) get_loggerz.ms_upload_progressc@seZdZdZdZdZdZdS) FileStatusz7Single-character status codes for compact JSON storage.ucfN)__name__ __module__ __qualname____doc__UPLOADED COMMITTEDFAILEDrr^/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/hub/upload_tracker.pyr s r c@s>eZdZdZdZdZdZdZdZdZ dZ e d e fd d Z d S) ErrorCategoryz:Classification of upload/commit errors for retry strategy.Ztransient_networkZtransient_serverZ throttledZ auth_failed not_foundZ file_invalidunknownreturncCs|tjtjtjfvSN)r AUTH_FAILED NOT_FOUND FILE_INVALIDselfrrr is_retryable.s zErrorCategory.is_retryableN)rrrrTRANSIENT_NETWORKTRANSIENT_SERVER THROTTLEDrrrUNKNOWNpropertyboolr"rrrrr$srerrorrcCsdt|}t|trtjSt|trtjSt|ttfr tj St|t j j rSt |dd}|durP|j}|dkr;tjS|dvrBtjS|dkrItjS|dkrPtjStjSt|tr}d|vr_tjSd|vsgd |vrjtjSd |vrqtjStd |rztjStjSt|ttfrd |vsd |vrtjSd|vsd|vrtjStj Sd|vsd|vrtj Sd|vrtj StjS)zClassify an exception into a retry category. Returns an ErrorCategory that indicates whether the error is transient (retryable) or permanent, and what kind of failure occurred. responseNi)iiiiZ429Z401Z403Z404z#(?:http[/\s]*)?5\d{2}|server.*errorz size changedz no such fileZ permissionz access deniedtimeoutz timed out connection)strlower isinstanceFileNotFoundErrorrrPermissionErrorConnectionError TimeoutErrorr#requests exceptions HTTPErrorgetattr status_coder%rrr$r& ValueErrorresearchIOErrorOSError)r)Z error_strrespstatusrrrclassify_error7sR      r@c @s6eZdZdZdeeefdefddZedede de d efd d Z dede de d e e fd d Zdede de de fddZdede de d efddZdede de d e efddZdede de fddZdeeee e ffddZ d*dede de defddZddZd d!Zd"d#Zd$e fd%d&Zd'd(Zd)S)+ UploadTrackeraUnified file-level upload tracker. Replaces both UploadHashCache (.ms_upload_cache) and UploadProgress (.ms_upload_progress) with a single file that tracks per-file hash and upload status. File format (version 3): { "version": 3, "repo_id": "user/repo", "files": { "path|mtime|size": {"hash": "...", "size": 123, "status": "c"}, ... } } Status values: "c" = committed (blob uploaded AND committed to repo) "u" = uploaded (blob uploaded, NOT yet committed) "f" = failed (no status field) = hash cached only, upload not attempted Thread safety: all mutations are protected by a lock. Persistence: atomic write via temp file + rename. cache_pathrepo_idcCs2t||_||_i|_t|_d|_|dSNF) r_path_repo_id_files threadingLock_lock_dirty_load)r!rBrCrrr__init__s   zUploadTracker.__init__rel_pathmtimesizercCs|d|d|S)zKBuild cache key from file metadata (same format as legacy UploadHashCache).|r)rNrOrPrrr _make_keyszUploadTracker._make_keycCsh||||}|j|j|}Wdn1swY|dus(d|vr*dS||d|ddS)zGet cached hash info for a file. Returns dict compatible with legacy UploadHashCache.get(): {'file_path_or_obj': rel_path, 'file_hash': ..., 'file_size': ...} or None if not cached or file has changed. NhashrP)Zfile_path_or_obj file_hash file_sizerRrJrGgetr!rNrOrPkeyentryrrrget_hashszUploadTracker.get_hash hash_infocCsp||||}|j$|j|i}|d|d<|d|d<||j|<d|_WddS1s1wYdS)zStore computed hash info for a file. Args: hash_info: dict with 'file_hash' and 'file_size' keys. rTrSrUrPTN)rRrJrGrWrK)r!rNrOrPr\rYrZrrrput_hashs   "zUploadTracker.put_hashcCsX||||}|j|j|}Wdn1swY|duo+|dtjkS)zsz&UploadTracker.save..)versionrCfilesFT)parentsexist_okz.tmp)dirsuffixwzutf-8)encoding) ensure_asciizFailed to save upload tracker: )rJrK_TRACKER_VERSIONrFrGitemsrEparentmkdirtempfilemkstempr-osfdopenjsondumpreplace BaseExceptionunlink Exceptionloggerwarning)r!datafdZtmp_pathrerrrsaves<     zUploadTracker.savec Csz |jjddWnty#}ztd|WYd}~nd}~ww|j|jd|_WddS1s;wYdS)zDelete the tracker file.T) missing_okzFailed to delete tracker file: NF) rErr=rrrJrGclearrK)r!rrrrrs "zUploadTracker.clearc Cs^|js |dSzt|jd }t|}Wdn1s"wYWntjtfyF}zt d|WYd}~dSd}~ww| d}|durW| |dS|t krgt d|dt d| dd }|r||j krt d |d |j d dS| d i|_tdd|jD}|dkrtdt|jd|d|dS)z8Load tracker state from disk, handling format migration.Nrz/Failed to load upload tracker, starting fresh: rkzUpload tracker version z is older than current z%. Data will be migrated on next save.rCrcz"Tracker repo_id mismatch (cached: z , current: z), ignoring stale tracker.rlcss$|] }|dtjkrdVqdS)r?N)rWr r)rgrrrr Ds z&UploadTracker._load..rzUpload tracker loaded: z entries, z committed.)rEexists_check_legacy_progressopenr|loadJSONDecodeErrorr=rrrW _migrate_v1rtrFrGsumvaluesinfolen)r!rrrrkZ stored_repoZcommitted_countrrrrL"sP      zUploadTracker._loadrcCspi}|D]\}}t|tr d|vr |d|ddd||<q||_d|_|r6tdt|ddSdS) adMigrate from legacy hash-only format (UploadHashCache v1). Old format: {"rel_path|mtime|size": {"file_hash": "...", "file_size": 123}} New format: {"rel_path|mtime|size": {"hash": "...", "size": 123}} Status is NOT set during migration -- cached hashes do not imply the file was committed (conservative approach). rTrUr)rSrPTz Migrated z' entries from legacy hash cache format.N) rur/rfrWrGrKrrr)r!rZmigratedrYvaluerrrrLs   zUploadTracker._migrate_v1cCs.|jjt}|rtd|ddSdS)z/Warn if legacy .ms_upload_progress file exists.z&Legacy upload progress file detected: z8. This file is no longer used. You may delete it safely.N)rErv_LEGACY_PROGRESS_FILErrr)r!Z legacy_pathrrrrcs  z$UploadTracker._check_legacy_progressNrc)rrrrrr-rrM staticmethodfloatintrRrrfr[r]r(r^r_r`rrrbrerrrLrrrrrrrAusH       * rAc @seZdZdZdedededdfddZdededed efd d Z dededede fd d Z dededefddZ dededefddZ ddZ ddedededefddZddZddZdS) NullTrackerzNo-op tracker for when caching is disabled. Implements the same interface as UploadTracker but does nothing, eliminating 'if tracker is not None' checks throughout api.py. rNrOrPrNcCdSrrr!rNrOrPrrrr[szNullTracker.get_hashr\cCrrr)r!rNrOrPr\rrrr]vszNullTracker.put_hashcCsdSrDrrrrrr^zrzNullTracker.is_committedcCrrrrrrrr_}rzNullTracker.get_statuscCrrrrrrrr`rzNullTracker.mark_uploadedcCrrr)r!rarrrrbrz NullTracker.mark_committed_batchrcrdcCrrr)r!rNrOrPrdrrrreszNullTracker.mark_failedcCrrrr rrrrrzNullTracker.savecCrrrr rrrrrzNullTracker.clearr)rrrrr-rrr[rfr]r(r^r_r`rbrerrrrrrrls,   r)rrzr:rxrHenumrpathlibrtypingrrrrrr|r4Zmodelscope.utils.loggerr rrrtr r-rrr@rArrrrrs(   >x