o Oc\O @sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl ZddlmZddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlm Z ddl!m"Z"ddl#m$Z$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z/ddl0m1Z1ddl2m3Z3m4Z4ddl5m6Z6ddl7m8Z8m9Z9m:Z:erddlm;Z;neZ?ej@jAjBZCeeDeDfZEdeDdeeDfddZFGdddeGZHde"ddfddZIGdddeGZJdeDde-ddfd d!ZKdeDde-de"fd"d#ZLd$eEdeeDfd%d&ZMd'eDdeDfd(d)ZNd'eDdeDfd*d+ZOe Pd,e jQZRd-eDd.eSdeDfd/d0ZTdeDdeDfd1d2ZUd3eeDeeDfd4eDd5eDdee)fd6d7ZVGd8d9d9ZWGd:d;d;e;ZXdZYeYd?d@dee)fdAdBZZGdCd@d@Z[GdDdEdEeZ\ dVdFe)dGeeDeGfdHeedIddfdJdKZ] LdWde"dMeSde[fdNdOZ^ dVdFe)dee-ded@fdPdQZ_GdRdSdSeZ`GdTdUdUZadS)XzO The main purpose of this module is to expose LinkCollector.collect_sources(). N) HTMLParser)Values) TYPE_CHECKINGCallableDictIterableListMutableMapping NamedTupleOptionalSequenceTupleUnion)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_file)pairwiseredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_source)ProtocolurlreturncCs6tjD]}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rZschemeslower startswithlen)r!schemer'?D:\Flask\env\lib\site-packages\pip\_internal\index\collector.py_match_vcs_scheme:s r)cs*eZdZdededdffdd ZZS)_NotAPIContent content_type request_descr"Ncst||||_||_dSN)super__init__r+r,)selfr+r, __class__r'r(r/Fs z_NotAPIContent.__init__)__name__ __module__ __qualname__strr/ __classcell__r'r'r1r(r*Es"r*responsecCs2|jdd}|}|drdSt||jj)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr#r$r*requestmethod)r8r+content_type_lr'r'r(_ensure_api_headerLsrAc@s eZdZdS)_NotHTTPN)r3r4r5r'r'r'r(rBbsrBsessioncCsFtj|\}}}}}|dvrt|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrBheadrrA)r!rCr&netlocpathqueryfragmentrespr'r'r(_ensure_api_responsefs  rPcCsxtt|jr t||dtdt||j|dgdddd}t |t |tdt||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rCzGetting page %sz, )r;z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r<zFetched page %s as %sr9r:) rrfilenamerPloggerdebugrr=joinrrAr<)r!rCrOr'r'r(_get_simple_responsexs&  rWr<cCs<|rd|vrtj}|d|d<|d}|rt|SdS)z=Determine if we have any encoding information in our headers.r9z content-typecharsetN)emailmessageMessage get_paramr6)r<mrXr'r'r(_get_encoding_from_headerss    r^partcCtjtj|S)zP Clean a "part" of a URL path (i.e. after splitting on "@" characters). )rGrHquoteunquoter_r'r'r(_clean_url_path_partsrdcCr`)z Clean the first part of a URL path that corresponds to a local filesystem path (i.e. the first part after splitting on "@" characters). )rGr> pathname2url url2pathnamercr'r'r(_clean_file_url_paths rgz(@|%2F)rL is_local_pathcCs^|rt}nt}t|}g}tt|dgD]\}}|||||qd |S)z* Clean the path portion of a URL. ) rgrd_reserved_chars_resplitr itertoolschainappendupperrV)rLrhZ clean_funcpartsZ cleaned_partsZto_cleanreservedr'r'r(_clean_url_paths  rrcCs6tj|}|j }t|j|d}tj|j|dS)z Make sure a link is fully quoted. For example, if ' ' occurs in the URL, it will be replaced with "%20", and without double-quoting other characters. )rh)rL)rGrHurlparserKrrrL urlunparse_replace)r!resultrhrLr'r'r( _clean_links rwelement_attribspage_urlbase_urlcCsL|d}|s dSttj||}|d}|d}t||||d}|S)zW Convert an anchor element's attributes in a simple repository page to a Link. hrefNzdata-requires-pythonz data-yanked) comes_fromrequires_python yanked_reason)r=rwrGrHurljoinr)rxryrzr{r!Z pyrequirer~linkr'r'r(_create_link_from_elements   rc@s6eZdZd ddZdedefdd Zdefd d ZdS) CacheablePageContentpage IndexContentr"NcCs|jsJ||_dSr-)cache_link_parsingrr0rr'r'r(r/s  zCacheablePageContent.__init__othercCst|t|o|jj|jjkSr-) isinstancetyperr!)r0rr'r'r(__eq__szCacheablePageContent.__eq__cCs t|jjSr-)hashrr!r0r'r'r(__hash__"s zCacheablePageContent.__hash__)rrr"N) r3r4r5r/objectboolrintrr'r'r'r(rs rc@s"eZdZdddeefddZdS) ParseLinksrrr"cCsdSr-r'rr'r'r(__call__'szParseLinks.__call__N)r3r4r5rrrr'r'r'r(r&srfncsPtjdddtdttffdd tdddttffd d }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. N)maxsizecacheable_pager"cst|jSr-)listr)r)rr'r(wrapper2sz*with_cached_index_content..wrapperrrcs|jr t|St|Sr-)rrr)rrrr'r(wrapper_wrapper6s  z2with_cached_index_content..wrapper_wrapper) functools lru_cacherrrwraps)rrr'rr(with_cached_index_content+s rrrc cs |j}|drSt|j}|dgD]9}|d}|dur#q|d}|r2t|ts2d}n|s6d}t t t j |j||j|d||did VqdSt|j}|jp\d }||j||j}|jpm|} |jD]} t| || d } | durqq| VqqdS) z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r;filesr!NZyankedrizrequires-pythonhashes)r|r}r~rzutf-8)ryrz)r+r#r$jsonloadscontentr=rr6rrwrGrHrr!HTMLLinkParserencodingfeeddecoderzanchorsr) rr@datafileZfile_urlr~parserrr!rzanchorrr'r'r( parse_links?sH           rc @sHeZdZdZ ddededeedededd f d d Zdefd d Z d S)rz5Represents one response (or page), along with its URLTrr+rr!rr"NcCs"||_||_||_||_||_dS)am :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. N)rr+rr!r)r0rr+rr!rr'r'r(r/rs  zIndexContent.__init__cCs t|jSr-)rr!rr'r'r(__str__s zIndexContent.__str__T) r3r4r5__doc__bytesr6r rr/rr'r'r'r(ros" csveZdZdZdeddffdd Zdedeeeeefddfd d Z deeeeefdeefd d Z Z S) rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. r!r"Ncs$tjdd||_d|_g|_dS)NT)Zconvert_charrefs)r.r/r!rzr)r0r!r1r'r(r/s zHTMLLinkParser.__init__tagattrscCsR|dkr|jdur||}|dur||_dSdS|dkr'|jt|dSdS)Nbasea)rzget_hrefrrndict)r0rrr{r'r'r(handle_starttags  zHTMLLinkParser.handle_starttagcCs"|D] \}}|dkr|SqdS)Nr{r')r0rnamevaluer'r'r(rs zHTMLLinkParser.get_href) r3r4r5rr6r/rr r rrr7r'r'r1r(rs &.rrreasonmeth).NcCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rTrU)rrrr'r'r(_handle_get_simple_failsrTrcCs&t|j}t|j|jd||j|dS)Nr9)rr!r)r^r<rrr!)r8rrr'r'r(_make_index_contents rc Cs |durtd|jddd}t|}|r td||dStj|\}}}}}}|dkrPt j tj |rP|dsC|d7}tj|d}td |zt||d }Wntyhtd |YdSty}ztd ||j|jWYd}~dSd}~wty}z t||WYd}~dSd}~wty}z t||WYd}~dSd}~wty}zd } | t|7} t|| tjdWYd}~dSd}~wtjy}zt|d|WYd}~dSd}~wtjyt|dYdSwt||j dS)Nz?_get_html_page() missing 1 required keyword argument: 'session'#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %srQz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)r)! TypeErrorr!rkr)rTwarningrGrHrsosrLisdirr>rfendswithrrUrWrBr*r,r+rrrrr6inforConnectionErrorTimeoutrr) rrCr!Z vcs_schemer&_rLrOexcrr'r'r(_get_index_contentsv       rc@s.eZdZUeeeed<eeeed<dS)CollectedSources find_links index_urlsN)r3r4r5r r r__annotations__r'r'r'r(rs rc @seZdZdZdededdfddZe dded ed e ddfd d Z e de e fd dZdedeefddZde dedefddZdS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rC search_scoper"NcCs||_||_dSr-)rrC)r0rCrr'r'r(r/s zLinkCollector.__init__Foptionssuppress_no_indexcCs`|jg|j}|jr|stdddd|Dg}|jp g}tj||d}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr-)r).0r!r'r'r( (sz'LinkCollector.create..rr)rCr) index_urlextra_index_urlsno_indexrTrUrVrrcreater)clsrCrrrrrlink_collectorr'r'r(rs"  zLinkCollector.createcCs|jjSr-)rrrr'r'r(r9szLinkCollector.find_linkslocationcCst||jdS)z> Fetch an HTML page containing package links. rQ)rrC)r0rr'r'r(fetch_response=szLinkCollector.fetch_response project_namecandidates_from_pagecstfddj|D}tfddjD}ttj rIddt ||D}t |d|dg|}t d|tt|t|d S) Nc3&|]}t|jjdddVqdS)FrZpage_validatorZ expand_dirrNrrCZis_secure_originrlocrr0r'r(rI z0LinkCollector.collect_sources..c3r)TrNrrrr'r(rSrcSs*g|]}|dur|jdurd|jqS)Nz* )r)rsr'r'r( _s z1LinkCollector.collect_sources..z' location(s) to search for versions of : r) collections OrderedDictrZget_index_urls_locationsvaluesrrT isEnabledForloggingDEBUGrlrmr%rUrVrr)r0rrZindex_url_sourcesZfind_links_sourceslinesr'rr(collect_sourcesCs2       zLinkCollector.collect_sources)F)r3r4r5rrrr/ classmethodrrrpropertyrr6rrr rrrrrr'r'r'r(rs<   rr-r)brr email.messagerYrrlrrrre urllib.parserGurllib.requestZxml.etree.ElementTreexmlZ html.parserroptparsertypingrrrrrr r r r r r pip._vendorrZpip._vendor.requestsrZpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkrZ!pip._internal.models.search_scoperpip._internal.network.sessionrZpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrrpip._internal.vcsrZsourcesrrrr r getLoggerr3rTZetreeZ ElementTreeZElementZ HTMLElementr6ZResponseHeadersr) Exceptionr*rArBrPrWr^rdrgcompile IGNORECASErjrrrrwrrrrrrrrrrrrr'r'r'r(s  4             ?   /      D