\`f>pddlZddlZddlZddlZddlZddlZddlmZmZm Z ddl m Z ddl m Z ddlmZddlmZddlmZddlmZmZmZmZmZmZmZdd lmZmZmZddl Z dd l m!Z!dd l"m#Z#ejHe%Z&d Z'd Z(dZ)dedfdZ*GddZ+Gdde+Z,GddZ-Gdde.Z/dZ0 d8de1de-fdZ2 d9ded efd!e3d"e4d#ejjd$ee6def d%Z7 d:ded efd'ee3d(e6d"e4def d)Z8e6d*de&jrdddddddd&f d+e6d"ee6d,ed-eed.ee6d/eed0eeee6ge6fd1e3d2e1d3e6fd4Z:Gd5d6Z; d9d7Z.combine_single/sY(3-( !}  c* CKD c&k-- 1 *%%r#)baseadd_onsr-rr*s r! combine_urlr1.s.& C*S&)* Jr#return) FileResponse UrlResponsec |j}|jdrd|z}|jjdrh|jdrtj d||t dd} t|d5}|j}dddt| St|fi|S#1swY"xYw#t$r:}|j}|jtk(rt}t||d||d}~wwxYw) a0Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. In the case of parameter passthrough to readurl, default values for some parameters. See: call-signature of readurl in this module for param docs. r&z file://%szfile://dataz'Unable to post data to file resource %sNrb)causecodeheadersr)contents)lstrip startswithlowergetLOGwarninglenopenreadIOErrorerrnorr UrlErrorr3readurl)rkwargs file_pathfpr;er9s r!read_file_or_urlrM>s **,C ~~cC yy{i( ::f  KKA3 GI()  Mi& %"779 %I99s%f%% % % M77Dww&  tE1 L  Ms0< CB:C:C?C D 5DD c eZdZddZdZdZy)StringResponsec<||_i|_||_d|_yN)r9r:r;r)selfr;r9s r!__init__zStringResponse.__init__^s    r#c$|jdk7ryy)NFTr9rRargsrIs r!okzStringResponse.okds 99 r#c8|jjdS)Nzutf-8)r;decoderRs r!__str__zStringResponse.__str__is}}##G,,r#NrU)__name__ __module__ __qualname__rSrYr]r.r#r!rOrO]s  -r#rOceZdZddZy)r3cBtj|||||_y)NrV)rOrSr)rRr,r;r9s r!rSzFileResponse.__init__nshT:r#Nr^r_r`rarSr.r#r!r3r3msr#r3ceZdZdejfdZedefdZedZ ddZ edZ edZ d Z dd eed edeefd Zy )r4responsec||_yrQ) _response)rRrfs r!rSzUrlResponse.__init__ts !r#r2c\|jjy|jjS)Nr#)rhcontentr\s r!r;zUrlResponse.contentsws% >> ! ! )~~%%%r#c.|jjSrQ)rhrr\s r!rzUrlResponse.url}s~~!!!r#c@d}|rd}d|jcxkr|kryyy)Ni,irUTFrV)rR redirects_okuppers r!rYzUrlResponse.oks- E $)) #e # $r#c.|jjSrQ)rhr:r\s r!r:zUrlResponse.headerss~~%%%r#c.|jjSrQ)rh status_coder\s r!r9zUrlResponse.codes~~)))r#c.|jjSrQ)rhtextr\s r!r]zUrlResponse.__str__s~~"""r# chunk_sizedecode_unicodec#XK|jj||Ed{y7w)atIterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. :param chunk_size: Number of bytes it should read into memory. :param decode_unicode: If True, content will be decoded using the best available encoding based on the response. N)rh iter_content)rRrtrus r!rwzUrlResponse.iter_contents">>..z>JJJs *(*N)F)rF)r_r`rarequestsResponserSpropertybytesr;rrYr:r9r]rintboolr rwr.r#r!r4r4ss"!2!2"&%&& ""&&**#EJ K"3- K=A K % Kr#r4ceZdZddZy)rGNctj|t|||_||_||_|j i|_||_yrQ)rErSr)r8r9r:r)rRr8r9r:rs r!rSzUrlError.__init__sCs5z*   << DLr#)NNNrdr.r#r!rGrGsr#rGci}t|j}|dk(rE|rCd|vr|dr |d|d<nd|d<d|vrd|vr|d|dg|d<|Sd|vrt|d|d<|S)Nhttpsca_certsverifyT cert_filekey_filecert)rrr))r ssl_detailsssl_argsrs r! _get_ssl_argsrsH c] ! !F [  $Z)@!,Z!8HX !%HX  + %* *CK(J' HV  OK '";{#;._cbsNr#r6r:z'[%s/%s] open '%s' with %s configurationinfinitez(Read from %s (%s, %sb) after %s attempts)rrfrqr9r:r1Please wait %s seconds while we wait to try againz"This path should be unreachable...r.)%r"updater isinstancetuplemaxfloatr|rversion_stringrxSessionr itemsr?copydeepcopyrr@debugrequestraise_for_statusrqrBrjr4rSSLErrorrGRequestException HTTPErrorhasattrrfr:timesleep RuntimeError)rr6rretries sec_betweenr: headers_cbheaders_redactr check_statusr exception_cbsessionr log_req_resprequest_methodrreq_args manual_tries def_headersrifiltered_req_argskvmatched_headerskeyrrL url_error will_retrys ` r!rHrHsjh C.CH OOM#{34"1H  #'U'HX gu %")HY "%eGna"8HY L3w ( ="*J %  +(+A""$ II: AIIQ  q> !ML(\ ; <|r|j|ry|||S)z Execute func with optional delayrNwaitrrrrrs r!_run_func_with_delayrrs' ::e: $  g r#333333? addresses stagger_delayc 2d}d}d}g}tj}tt|} t |D cic]$\} } | j t || ||| |z| &} } } t| |D]h} | | }| j}|r|}|j|/| j}|sB|j||fc| jdS|rtjd|||tjd|t!d cc} } w#t"$r8tjd d j%|d j%|YnwxYw | jd||fS#| jdwxYw) aexecute multiple callbacks in parallel Run blocking func against two different addresses staggered with a delay. The first call to return successfully is returned from this function and remaining unfinished calls are cancelled if they have not yet started N) max_workersrrFrzE(%F'E((FFinfmax_wait status_cbr sleep_timer sleep_time_cbrconnect_synchronously async_delayc n dtdtffd } ddtdtdtfdd fd  fd fd } fd }tj}r |r td| r| n|}|xs| }dd} ||}|||||}|r|\}}||jfS||rn`dzt j d|tj|tj}|r||z|zkDr||z z }|dkrnt jdy)a1 urls: a list of urls to try max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. a number <= 0 will always result in only one try timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers for request. headers_redact: a list of header names to redact from the log sleep_time: Amount of time to sleep between retries. If this and sleep_time_cb are None, the default sleep time defaults to 1 second and increases by 1 seconds every 5 tries. Cannot be specified along with `sleep_time_cb`. exception_cb: call method with 2 arguments 'msg' (per status_cb) and 'exception', the exception that occurred. sleep_time_cb: call method with 2 arguments (response, loop_n) that generates the next sleep time. Cannot be specified along with 'sleep_time`. request_method: indicate the type of HTTP request, GET, PUT, or POST connect_synchronously: if false, enables executing requests in parallel async_delay: delay before parallel metadata requests, see RFC 6555 returns: tuple of (url, response contents), on failure, (False, None) the idea of this routine is to wait for the EC2 metadata service to come up. On both Eucalyptus and EC2 we have seen the case where the instance hit the MD before the MD service was up. EC2 seems to have permanently fixed this, though. In openstack, the metadata service might be painfully slow, and unable to avoid hitting a timeout of even up to 10 seconds or more (LP: #894279) for a simple GET. Offset those needs with the need to not hang forever (and block boot) on a system where cloud-init is configured to look for EC2 Metadata service but is not going to find one. It is possible that the instance data host (169.254.169.254) may be firewalled off Entirely for a system, meaning that the connection will block forever unless a timeout is set. The default value for max_wait will retry indefinitely. loop_numberr2cS|dzdzS)Nrr.)_rrs r!default_sleep_timez(wait_for_url..default_sleep_times'3zM9IA9MMr#rr start_timercj|tddfvry|dkxstj|z |z|kDS)z4Check if time is up based on start time and max waitrNFr)rr)rrrs r!timeupzwait_for_url..timeups? e d+ +A  IIK* $z 1H < r#cF|js?d|jz}tt||j|j|}||fS|j s?d|jz}tt||j|j|}||fSd}d}||fS)z?Map requests response code/contents to internal "UrlError" typezempty response [%s]rzbad status code [%s]rN)r;r9rGrr:rY)rfrreasonurl_excs r!handle_url_responsez)wait_for_url..handle_url_responses  *hmm.read_url_handle_exceptions2s )$/MC1(C@OGVX&z12 +3ux'  877E<8 8     z  zW 5  )A-FG ,q0FG s#B B3 B B3"B..B3c:t|in||dS)NF)r:rrrr)rH)rrrrrs r! read_url_cbz!wait_for_url..read_url_cbRs- $,B*S/))   r#c fd} D]X}tj}dk7r+ |ry r|z| zkDrt| z|z  |||||}|sV|cSy)z|iterate over list of urls, request each one and handle responses and thrown exceptions individually per url c||fSrQr.)rrrs r!url_reader_serialz@wait_for_url..read_url_serial..url_reader_serialasS'23 3r#rN)rr|)rrrrr rnowoutloop_nrrrrrs ` r!read_url_serialz%wait_for_url..read_url_serial\s  4 C))+C{(J/(w*x*?@":#8C"?@G,!3 FFC # r#cLtt|}| |||}|r|Sy)zpass list of urls to dual_stack which sends requests in parallel handle response and exceptions of the first endpoint to respond )rrN)rr) rrrrurl_reader_parallelr rrrrs r!read_url_parallelz'wait_for_url..read_url_parallelwsB&  %   ) z66  J r#z3sleep_time and sleep_time_cb are mutually exclusiveNrrz$Timed out, no response from urls: %s)FN)r) r|rrrr;r@rrr)rrrrrrrrrrrrrrrr do_read_urlcalculate_sleep_timerfcurrent_sleep_timeraddress current_timerr rrrs`` ``` ` ` @@@@@r! wait_for_urlrsdrN3N5N  E u !.6@ 6 JmNOO16G)>,>FH 1(FC*g|YG  # GXX../ / (J(: ; ! ?   %&yy{ |g- X0EE,";>$2E2E#Fd))4 %yy} % % %s A--A6c|jsy|j}|i}|||<t|jd5}|jt j |dddy#1swYyxYw)Nwr))rr$rCwriter-dumps)rRhostvaluecurrKs r!update_skew_filezOauthUrlHelper.update_skew_filesj"" !!# ;CD $%%C 0 &B HHTZZ_ % & & &s %A,,A5ct|tr|jdk(s|jdk(syd|jvr!tj d|jy|jd} t jt|}t|t j z }t|jj}|jj|d}t!||z |j"kDr)|j%||tj d||||j|<y#t$r!}tj d||Yd}~yd}~wwxYw)Niidatez$Missing header 'date' in %s responsez#Failed to convert datetime '%s': %srz$Setting oauth clockskew for %s to %d)rrGr9r:r@rArmktimerrr|rrnetlocr%r?absr!r6) rRrrr8 remote_timerLskewr3old_skews r!rzOauthUrlHelper.exception_cbs! y( +3&)..C*?  ** * KK>  O   ( ++io6K ;,- &-->>%%dA. x$ $"8"8 8  ! !$ - KK>d K#t  KK=tQ G  s/D<< E&E!!E&cL|jsiSd}t|j}|jr;||jvr-t t j |j|z}t ||j|j|j|j|S)N)rrrrr timestamp) r rr:r%r|r oauth_headersrrrr)rRrr@r3s r!rzOauthUrlHelper.headers_cbs~~I }## >>ddnn4DIIK(4>>$+??I**nn** 00   r#ct|j|jd|d<t|j|jd|d<||i|S)Nrr)r _headers_cbr? _exception_cb)rR wrapped_funcrXrIs r!_wrappedzOauthUrlHelper._wrappeds\&   fjj6 |")    > :" ~T,V,,r#c0|jt||SrQ)rFrrWs r!rzOauthUrlHelper.wait_for_urls}}\488r#c0|jt||SrQ)rFrHrWs r!rHzOauthUrlHelper.readurls}}WdF33r#ctd} |r |||}|j|||S#|j||wxYwrQ)r)rRextra_exception_cbrrrets r!rDzOauthUrlHelper._exception_cb sD .!(i8   c9 -    c9 -s #7c^i}|r||}|j|j||SrQ)rr)rRextra_headers_cbrr:s r!rCzOauthUrlHelper._headers_cb)s. &s+Gts+,r#)NNNNz/run/oauth_skew.json) r_r`rarSr$r6rrrFrrHrDrCr.r#r!rrsE- #6 &8 $-94r#rc ddlm}|r t |}nd}|j |||||j |}|j|\} } } | S#t$r}td|d}~wwxYw)Nrzoauth support is not available) client_secretresource_owner_keyresource_owner_secretsignature_methodr@)oauthlib.oauth1oauth1 ImportErrorNotImplementedErrorr)ClientSIGNATURE_PLAINTEXTsign) rrrrrr@rTrLclient_urisigned_headers_bodys r!rArA1sK( N  ]]%$*33 F#)++c"2D.% # K!"BCJKsA A* A%%A*)NNrrNNNNTTNNFTrFrQ)r )=rr-loggingr+rrconcurrent.futuresrrr email.utilsrrFr functoolsr http.clientr itertoolsr typingr r r rrrr urllib.parserrrrxr cloudinitr getLoggerr_r@rr"r1rMrOr3r4rErGrr}rHr)r|rrrrrrrrAr.r#r!risz   MM!!HHH44g! "  &u-J'K&> - - > .K.Kbw(      #n=" #n=$%n=l"  38   ??  E?   *! P- 38 P-CyP-P- P-  P-jEl#))%)"&'+=A"&\\e_\ \ " \\8$\Hc5\5%89:\\ \\~xxxLPr#