a +b @sldZddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZdd lm Z m!Z!zdd lm"Z"e"j#Wne$ydZ"Yn0iZ%e&e%Z&e&d d e"rd nddddZ'dZ(dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dddddd d!d"e-gZ4dGd#d$Z5d%d&Z6Gd'd(d(ej7Z8ej9Gd)d*d*e:Z;eeZJd?d@ZKdAdBZLdCdDZMdEdFZNdS)Hamstore repository data in SQLite (EXPERIMENTAL) The sqlitestore extension enables the storage of repository data in SQLite. This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY GUARANTEES. This means that repositories created with this extension may only be usable with the exact version of this extension/Mercurial that was used. The extension attempts to enforce this in order to prevent repository corruption. In addition, several features are not yet supported or have known bugs: * Only some data is stored in SQLite. Changeset, manifest, and other repository data is not yet stored in SQLite. * Transactions are not robust. If the process is aborted at the right time during transaction close/rollback, the repository could be in an inconsistent state. This problem will diminish once all repository data is tracked by SQLite. * Bundle repositories do not work (the ability to use e.g. `hg -R log` to automatically overlay a bundle on top of the existing repository). * Various other features don't work. This extension should work for basic clone/pull, update, and commit workflows. Some history rewriting operations may fail due to lack of support for bundle repositories. To use, activate the extension and set the ``storage.new-repo-backend`` config option to ``sqlite`` to enable new repositories to use SQLite for storage. )absolute_importN_)nullrevsha1nodeconstantsshort)attr) ancestordagopencodingerror extensions localrepomdiffpycompat registrar requirementsutilverify) repositoryr)hashutil storageutil)zstdstoragesqlite.compressionzstdzlibT)defaultZ experimentalsships-with-hg-coresexp-sqlite-001sexp-sqlite-comp-001=zstdsexp-sqlite-comp-001=zlibsexp-sqlite-comp-001=nonesexp-sqlite-shallow-fileszCREATE TABLE delta ( id INTEGER PRIMARY KEY, compression INTEGER NOT NULL, hash BLOB UNIQUE ON CONFLICT ABORT, delta BLOB NOT NULL )zKCREATE TABLE filepath ( id INTEGER PRIMARY KEY, path BLOB NOT NULL )z4CREATE UNIQUE INDEX filepath_path ON filepath (path)acCREATE TABLE fileindex ( id INTEGER PRIMARY KEY, pathid INTEGER REFERENCES filepath(id), revnum INTEGER NOT NULL, p1rev INTEGER NOT NULL, p2rev INTEGER NOT NULL, linkrev INTEGER NOT NULL, flags INTEGER NOT NULL, deltaid INTEGER REFERENCES delta(id), deltabaseid INTEGER REFERENCES fileindex(id), node BLOB NOT NULL )zJCREATE UNIQUE INDEX fileindex_pathrevnum ON fileindex (pathid, revnum)zBCREATE UNIQUE INDEX fileindex_pathnode ON fileindex (pathid, node)aCREATE VIEW filedata AS SELECT fileindex.id AS id, filepath.id AS pathid, filepath.path AS path, fileindex.revnum AS revnum, fileindex.node AS node, fileindex.p1rev AS p1rev, fileindex.p2rev AS p2rev, fileindex.linkrev AS linkrev, fileindex.flags AS flags, fileindex.deltaid AS deltaid, fileindex.deltabaseid AS deltabaseid FROM filepath, fileindex WHERE fileindex.pathid=filepath.idzPRAGMA user_version=%dcCs|djddgt|dt||gt|}g}d}|D]Z\} } } | }| tkrf|| } n.| t krt| } n | t krt | } n t d| | | qD||vr|||} n|} |t| |} t| tst| } | S)z&Resolve a delta chain for a file node.aWITH RECURSIVE deltachain(deltaid, baseid) AS ( SELECT deltaid, deltabaseid FROM fileindex WHERE pathid=? AND node=? UNION ALL SELECT fileindex.deltaid, deltabaseid FROM fileindex, deltachain WHERE fileindex.id=deltachain.baseid AND deltachain.baseid IS NOT NULL AND fileindex.id NOT IN ({stops}) ) SELECT deltachain.baseid, compression, delta FROM deltachain, delta WHERE delta.id=deltachain.deltaid,?)ZstopsNsunhandled compression type: %d)executeformatjoinlentuplelistkeysCOMPRESSION_ZSTD decompressCOMPRESSION_NONECOMPRESSION_ZLIBzlibSQLiteStoreErrorappendpopreverserZpatches isinstancebytes)dbpathidnodeZ revisioncachestopridszstddctxresdeltasZlastdeltabaseidZ deltabaseid compressiondeltaZbasetextfulltextr@7/usr/lib64/python3.9/site-packages/hgext/sqlitestore.pyresolvedeltachains8     rBcCsFz|d|||fjWStjy@|d|fdYS0dS)Nz=INSERT INTO delta (compression, hash, delta) VALUES (?, ?, ?)z!SELECT id FROM delta WHERE hash=?r)r$ lastrowidsqlite3ZIntegrityErrorfetchone)r6r=hashr>r@r@rA insertdeltasrGc@s eZdZdS)r0N)__name__ __module__ __qualname__r@r@r@rAr0sr0c@sTeZdZeZeZeZeZeZ eZ eZ eZ eZ dS) revisionentryN)rHrIrJribridrevr8p1revp2revp1nodep2nodelinkrevflagsr@r@r@rArK srK)slotsc@sheZdZeZeZeZeZeZ eZ eZ eZ eZ eZejddZdS)sqliterevisiondeltaNr)rHrIrJrrLr8rQrRZbasenoderTZbaserevisionsizerevisionr>sidedataZprotocol_flagslinknoder@r@r@rArVsrV)frozenc@s0eZdZejddZejddZejddZdS) sqliteproblemNrW)rHrIrJrrLZwarningr r8r@r@r@rAr\(s  r\c@sReZdZdZddZddZddZdd Zd d ZdOddZ ddZ ddZ ddZ ddZ ddZddZddZddZd d!ZdPd"d#Zd$d%Zd&d'ZdQd*d+Zd,d-Zd.d/Zd0d1Zd2d3Zd d(d(ejd fd4d5Zd6d7ZdRd8d9ZdSd:d;Z dTd=d>Z!d?d@Z"dAdBZ#dCdDZ$dUdEdFZ%dVdGdHZ&dIdJZ'dWdKdLZ(dXdMdNZ)d S)Ysqlitefilestorez2Implements storage for an individual tracked path.cCsxtj|_||_||_d|_i|_i|_i|_t d|_ ||_ |dkr`t j dd|_t |_n d|_d|_|dS)N rr )level)rnullid_db_path_pathid _revtonode _nodetorev _revisionsrZ lrucachedict_revisioncache _compenginerZZstdCompressor_cctxZZstdDecompressor_dctx _refreshindex)selfr6pathr=r@r@rA__init__4s  zsqlitefilestore.__init__c Csi|_i|_i|_t|jd|jf}|s6d|_dS|dd|_|jd|jf}t|D]\}}|\}}}}}} } ||krt t d|t krt j } n |j|} |t krt j } n |j|} t|||||| | | | d } ||j|<||j|<| |j|<q^dS)Nz$SELECT id FROM filepath WHERE path=?rzgSELECT id, revnum, node, p1rev, p2rev, linkrev, flags FROM fileindex WHERE pathid=? ORDER BY revnum ASCs1sqlite database has inconsistent revision numbers rMrNr8rOrPrQrRrSrT)rdrerfr)rar$rbrc enumerater0rrrr`rK)rlr;irowrMrNr8rOrPrSrTrQrRentryr@r@rArkOsT    zsqlitefilestore._refreshindexcCs t|jSN)r'rfrlr@r@rA__len__szsqlitefilestore.__len__cCsttt|jSrt)iterrZxranger'rfrur@r@rA__iter__szsqlitefilestore.__iter__cCs|tjkrdS||jvSNF)rr`rerlr8r@r@rAhasnodes zsqlitefilestore.hasnoderNcCstjt|j||dS)N)startstop)rZiterrevsr'rf)rlr|r}r@r@rArevss zsqlitefilestore.revscCsJ|tjkrtjtjfS||jvr4t||jtd|j|}|j|jfSNno node) rr`rfr LookupErrorrbrrQrR)rlr8rsr@r@rAparentss     zsqlitefilestore.parentscCs>|tkrttfS||jvr"t||j|j|}|j|jfSrt)rrd IndexErrorrfrOrPrlrNrsr@r@rA parentrevss  zsqlitefilestore.parentrevscCs6|tjkrtS||jvr,t||jtd|j|Sr)rr`rrer rrbrrzr@r@rArNs   zsqlitefilestore.revcCs*|tkrtjS||jvr t||j|Srt)rrr`rdrrlrNr@r@rAr8s  zsqlitefilestore.nodecCst|||jSrt)rZ fileidlookuprbrzr@r@rAlookupszsqlitefilestore.lookupcCs4|tkr tS||jvrt||j|j|}|jSrt)rrdrrfrSrr@r@rArSs  zsqlitefilestore.linkrevcCs4|tkr dS||jvrt||j|j|jt@Sry)rrdrrfrT FLAG_CENSOREDrr@r@rA iscensoreds  zsqlitefilestore.iscensoredcCs2||}||}t|j||}t|j|Srt)rNr commonancestorsheadsrrZmaplistr8)rlZnode1Znode2Zrev1Zrev2Z ancestorsr@r@rArs  z$sqlitefilestore.commonancestorsheadscCst||j|jSrt)r Zdescendantrevsr~r)rlr~r@r@rA descendantsszsqlitefilestore.descendantscst|dur |dur ts tjgS|dur2|nt}fdd|pFgD}tjjj||d}fdd|DS)Ncsh|]}|qSr@rN.0nrur@rA z(sqlitefilestore.heads..)startrevstoprevscsg|]}|qSr@)r8)rrNrur@rA rz)sqlitefilestore.heads..) r'rr`rNrr Zheadrevssubsetr~r)rlr|r}rrr~r@rurAheadss zsqlitefilestore.headscCs.||}|jd|j||f}dd|DS)NzZSELECT node FROM filedata WHERE path=? AND (p1rev=? OR p2rev=?) ORDER BY revnum ASCcSsg|] }|dqS)rr@)rrrr@r@rArrz,sqlitefilestore.children..)rNrar$rb)rlr8rNr;r@r@rAchildrens    zsqlitefilestore.childrencCsN|tkr dS||jvrt||j|}||r@t||St||S)Nr)rrdrrenamedr'readrX)rlrNr8r@r@rAsizes   zsqlitefilestore.sizeFTcs|tjtfvrdSt|tr&|}|jvrDt|j t d|j vrXj |Sfddj D}|sxd|d<t j j|j |jd}j|jttB@rd}|rƈ|||j |<|S)Nrrcsi|]}j|j|qSr@)rfrMrrur@rA $rz,sqlitefilestore.revision..r:F)rr`rr4intr8rer rrbrrgrBrarcrjrfrTFLAG_MISSING_P1FLAG_MISSING_P2 _checkhash)rlr8raw _verifyhashr9r?r@rurArXs2        zsqlitefilestore.revisioncOs|j|i|Srt)rX)rlargskwargsr@r@rArawdata=szsqlitefilestore.rawdatacCst||Srt)rZfiltermetadatarXrzr@r@rAr@szsqlitefilestore.readcCs t||Srt)rZfilerevisioncopiedrzr@r@rArCszsqlitefilestore.renamedcCst||| Srt)rZfiledataequivalent)rlr8r?r@r@rAcmpFszsqlitefilestore.cmpc cs|dvrtd|dd|D}|s,dS|jdddgt|t|jg|}i}|D],\} } |jd|j| f}|d || <qbt j |||t |j ||||d D] } | VqdS) N)snodesrslinearNs"unhandled value for nodesorder: %scSsg|]}|tjkr|qSr@)rr`rr@r@rArWrz1sqlitefilestore.emitrevisions..zISELECT revnum, deltaid FROM fileindex WHERE pathid=? AND node in (%s)r"r#z9SELECT revnum from fileindex WHERE pathid=? AND deltaid=?r)Z deltaparentfn revisiondataassumehaveparentrevisions deltamodesidedata_helpers) r ProgrammingErrorrar$r&r'r(rcrEr emitrevisionsrV __getitem__) rlZnodesZ nodesorderrrrrr;Z deltabasesrNdeltaidr>r@r@rArIs@   zsqlitefilestore.emitrevisionscCs6|s|drt||}||||||}||S)Ns ) startswithrpackmeta addrevisionr8)rlZfiledatameta transactionrSp1p2rNr@r@rAadds zsqlitefilestore.addc Csx|rttd|du} |p(t|||}| r>||||||j|} | durV| S|||||||} ||j|<| S)Ns flags not supported on revisions) r0rrhashrevisionsha1rreget_addrawrevisionrg) rlrrrSrrr8rTZ cachedeltaZ validatehashrNr@r@rArs     zsqlitefilestore.addrevisionc snd}|D]\\}} } } } } }}d}|tj@r4|tO}|tj@rHtd|r| tjkrn| sntj} |tO}| tjkr| stj} |tO} | }|t kr |rt d}tj| dd}t| |}| d|t||krtj| |t@s$t| |fddr$|tO}|| }|jvrj|}|jt@r| tjkr| |_j| |_|jtM_jd j| |j|jf|jt@r| tjkr| |_j| |_ |jtM_jd j| |j|jf|r| |d}q| tjkr,t!d | }d}n d}| | f}j"||||| | ||d }|rb||d}q| S) NTrsunhandled revision flags>lllF)rcst|Srt)r'r)xrur@rArz*sqlitefilestore.addgroup..z0UPDATE fileindex SET p1rev=?, flags=? WHERE id=?z0UPDATE fileindex SET p2rev=?, flags=? WHERE id=?r) storedeltarT)#rZREVISION_FLAG_CENSOREDrr0rr`r{rrrNrrstructcalcsizer'rrZreplacediffheaderr ZCensoredBaseErrorrbrZdeltaiscensoredrfrTrQrerOrar$rMrRrPZpatchr)rlr<Z linkmapperrZ addrevisioncbZduplicaterevisioncbZmaybemissingparentsemptyr8rrrZ deltabaser>Z wireflagsrYZ storeflagsZbaserevZhlenZoldlenZnewlenrSrstextrrNr@rurAaddgroups               zsqlitefilestore.addgrouprc Cstd|id}t|t||kr4ttd|jd|j |j f d}t |jd||f}|D]}|\}}} t |j|| iddi|jd} t| } |jd kr|j| } t} n>|jd krt| } t} n$|jd kr| } t} ntd |jt| t| kr| } t} t|j| | | }|jd ||fqnt|} t|jt| |}|j |j}|tO}|jd|||j|f|jd|f||j dS)Nscensoredrs5censor tombstone must be no longer than censored dataz(SELECT deltaid FROM fileindex WHERE id=?rzGSELECT id, pathid, node FROM fileindex WHERE deltabaseid=? OR deltaid=?rrrrnone unhandled compression engine: %sz;UPDATE fileindex SET deltaid=?, deltabaseid=NULL WHERE id=?zSUPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL WHERE pathid=? AND node=?zDELETE FROM delta WHERE id=?)!rrr'rr Abortrrar$rfrMrEr)rBrjrsha1digestrhricompressr+r/r.r-rrGrTrrcrkrgclear)rltrZ censornodeZ tombstoneZcensoreddeltaidZrowsrrrMr7r8r? deltahash deltablobr=rZtombstonedeltaidrTr@r@rAcensorrevisionsp          zsqlitefilestore.censorrevisioncs0t|tdfddDjjS)Nrcsg|]}|qSr@rrrur@rAr{rz1sqlitefilestore.getstrippoint..)rZresolvestripinfor'rrSr)rlminlinkr@rurA getstrippointws zsqlitefilestore.getstrippointcCs`t|s dS||\}}|t|kr*dS||D]}|jd|j||fq4|dS)Nz/DELETE FROM fileindex WHERE pathid=? AND node=?)r'rr~rar$rcr8rk)rlrrrNZ_ignoredr@r@rAstrips zsqlitefilestore.stripcCsgSrtr@rur@r@rAfilesszsqlitefilestore.filescCsiSrtr@)rlZ nodeorrevZ_dfr@r@rArYszsqlitefilestore.sidedatacs\i}|rg|d<|rg|d<|r,t|d<|rLtfddjD|d<|rXd|d<|S)Nsexclusivefiless sharedfilessrevisionscountc3s|]}t|VqdSrt)r'rX)rr8rur@rA sz.sqlitefilestore.storageinfo..s trackedsizes storedsize)r'sumre)rlZexclusivefilesZ sharedfilesZrevisionscountZ trackedsizeZ storedsizedr@rurA storageinfos   zsqlitefilestore.storageinfoc cst|d<|D]n}||}z||Wqtyz}z8ttdt||f|dV|d|WYd}~qd}~00qdS)Nsskipreadsunpacking %s: %s)r r8)setr8rX Exceptionr\rrr)rlstaterNr8er@r@rAverifyintegritys  zsqlitefilestore.verifyintegritycCs|dur|dur||\}}|t|||kr4dSz |j|=WntyRYn0t|rnt|j||t t d|jdS)Nsintegrity check failed on %s) rrrrgKeyErrorZiscensoredtextr ZCensoredNodeErrorrbr0r)rlr?r8rrr@r@rArs   zsqlitefilestore._checkhashc  Cs|jdur$|jd|jf} | j|_|rF|\} } t| trr|| } n,|} | tj krZ|} nt | | | |} | tj krd} n |j| j} t| } |jdkr|j| }t}n>|jdkrt| }t}n$|jdkr| }t}ntd|jt|t| kr| }t}t|j|| |}t|}|tj kr:t}n |j|}|tj krVt}n |j|}|jd|j|||||||| f j}t |||||||||d }||j|<||j!|<||j|<|S)Nz&INSERT INTO filepath (path) VALUES (?)rrrrzINSERT INTO fileindex ( pathid, revnum, node, p1rev, p2rev, linkrev, flags, deltaid, deltabaseid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)ro)"rcrar$rbrCr4rr8rr`rZtextdiffrXrNrfrMrrrrhrirr+r/r.r-r rr'rGrrerKrd)rlr8rrrSrrrrTr;rr>Zbaseidrrr=rrNrOrPrMrsr@r@rArs                  zsqlitefilestore._addrawrevision)rN)NN)FT)NrN)NNF)r)N)FFFFF)NN)Nr)*rHrIrJ__doc__rnrkrvrxr{r~rrrNr8rrSrrrrrrrXrrrrrZCG_DELTAMODE_STDrrrrrrrrrYrrrrr@r@r@rAr]0sn=         * 7 $ t ]    r]cs0eZdZddZfddZeddZZS)sqliterepositorycCsdSryr@rur@r@rAcancopyXszsqliterepository.cancopycsN}ttj|i|}|r&|Sjdfdd}|d||S)NzBEGIN TRANSACTIONcsjdSrt)_dbconncommitrrur@rAcommittransactionesz7sqliterepository.transaction..committransactions sqlitestore)Zcurrenttransactionsuperrrrr$Z addfinalize)rlrrcurrentrr __class__rurAr[s   zsqliterepository.transactioncCsFtj}|jr(|jd|kr(|jdSt|jd}||f|_|S)Nrrs db.sqlite) threadingcurrent_threadidentramakedbZsvfsr&)rltidr6r@r@rArls   zsqliterepository._dbconn)rHrIrJrrpropertyr __classcell__r@r@rrArWs rcCsttt|}t|_|dd}|dkrNtD]}||q4| n|t krXnt t d|d|S)z3Construct a database handle for a database at path.zPRAGMA user_versionrs(sqlite database has unrecognized versionzPRAGMA journal_mode=WAL)rDZconnectr Z strfromlocalr5Z text_factoryr$rE CREATE_SCHEMArCURRENT_SCHEMA_VERSIONr rr)rmr6r;Z statementr@r@rAr}s   rcCsF|ttr|t|t|t|t|tjdSrt) r REQUIREMENTrREQUIREMENT_ZSTDREQUIREMENT_ZLIBREQUIREMENT_NONEREQUIREMENT_SHALLOW_FILESrZNARROW_REQUIREMENT)uiZ supportedr@r@rA featuresetups     rcCs |ddkr|||Sd|vr,ttdhd}t||}|r`ttddt|d|d<|||}|t|d d }|d krt sttd |d kr|t n:|d kr|t n&|dkr|t nttd|| dr|t|S)Nbackendssqlites sharedrepos3shared repositories not supported with SQLite store>s narrowfilesrshallowfilestores6SQLite store does not support repo creation option: %ss, srevlogv1rrrsjstorage.sqlite.compression set to "zstd" but zstandard compression not available to this Mercurial installrrsDunknown compression engine defined in storage.sqlite.compression: %sr)r rrrr&sortedrrZconfigrrrrrr)origrZ createoptsZknownZ unsupportedrr=r@r@rAnewreporequirementssP             rc@seZdZdZddZdS)sqlitefilestoragez)Repository file storage backed by SQLite.cCsd|ddkr|dd}t|jvr(d}n.t|jvr8d}nt|jvrHd}nttdt|j||S)Nr/rrrrsEunable to determine what compression engine to use for SQLite storage) rrrrr rrr]r)rlrmr=r@r@rAfiles     zsqlitefilestorage.fileN)rHrIrJrrr@r@r@rArsrcKs8t|vr t|vr|tjtS|f||d|SdS)z=Produce a type conforming to ``ilocalrepositoryfilestorage``.)rfeaturesN)rrrrZ!REPO_FEATURE_SHALLOW_FILE_STORAGEr)rrrrr@r@rAmakefilestorages  rcKs8t|vr&t|vr"ts"ttdtS|fd|i|S)NsWrepository uses zstandard compression, which is not available to this Mercurial installr)rrrr rrr)rrrrr@r@rAmakemains rcOs ||g|Ri|d|_dSry)Zwarnorphanstorefiles)rrlrrr@r@rA verifierinitsrcCsJtjtttdtttdtttdttt j dt dS)Nsnewreporequirementssmakefilestoragesmakemains__init__) rZfeaturesetupfuncsrrr Z wrapfunctionrrrrZverifierr)rr@r@rAextsetups rcCst|trd|_dSrt)r4rra)rZrepor@r@rA reposetup)s r)N)OrZ __future__rrDrrr/Zmercurial.i18nrZmercurial.noderrrZmercurial.thirdpartyrZ mercurialr r r r r rrrrrrrZmercurial.interfacesrZ interfaceutilZmercurial.utilsrrr __version__ ImportErrorZ configtableZ configitemZ testedwithrrrrrrr-r+r.rrrrrBrGZ StorageErrorr0sobjectrKZ implementerZirevisiondeltarVZiverifyproblemr\Z ifilestorager]ZlocalrepositoryrrrrZilocalrepositoryfilestoragerrrrrrr@r@r@rAs&   8         ? A      ,& A