j_ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlmZedZdZdZhdZhdZgdZd d d d d dddZdZdZdd)Z,d?d,Z-d@d-Z.d.Z/d/Z0d0Z1d1Z2d2Z3d3Z4d4Z5d5Z6d6Z7d7Z8d8Z9d9Z:d@d:Z;ecsrfsigntokencookiecsrfid _tb_token_ loginpointid authorization> user-agent content-typex-requested-withacceptoriginreferer)scenecampaignbidworditem_promotioncrowdcreativeareau营销场景报表u 计划报表u关键词报表 商品报表u 人群报表u 创意报表u 地域报表c2t|dtjidS)Nfile)printsysstderrargss alimama_exporter/cli.pyeprintr" s 4!cj!!!!!cntjt|ddS)Nutf-8replace)encodingerrors)jsonloadsr read_textpaths r! load_jsonr.$s+ :d4jj**GI*NN O OOr#Fct|}|jdd|t j|ddd|rt j|d|S) NTparentsexist_okF ensure_asciiindentr%r'i)rparentmkdir write_textr)dumpsoschmod)r-datasecrets r! write_jsonr@(sn ::DKdT222OODJt%BBBWOUUU  u Kr#cd|pgDS)Nci|]T}|d|dd|ddUS)namevalue)getlower).0hs r! zheader_map..2sW b b baTUTYTYZ`TaTa bAEE&"   # # % %quuWb'9'9 b b br#headerss r! header_maprN1s b b7=b b b bbr#ci}|pddD]M}d|vr|dd\}}|}|r|||<N|S)NrD;=)splitstrip) cookie_headeroutpartkvs r!parse_cookie_headerrZ5sx C$"++C00 d?? zz#q!!1 GGII  WWYYCF Jr#c<t|t|tttz}fd|D}ttt|t|||k||dS)Nclg|]0}||k.|1SrKrF)rHrXabs r! z"cookie_compare..Es7 7 7 71!%%((aeeAhh"6"6Q"6"6"6r#)a_cookie_countb_cookie_countcommon_cookie_names_countequal_name_value_countidentical_cookie_headercommon_cookie_namesequal_cookie_names)rZsortedsetlen)a_headerb_headercommonequalr^r_s @@r!cookie_compareroAsH%%AH%%A CFFSVVO $ $F 7 7 7 7 7 7 7 7Ea&&a&&%([["%e**#+x#7%#  r#ct|}t|trU|dp?|dp*|didpdSdS)NCookierrMrD)r. isinstancedictrFr-r>s r!read_cookie_jsonruQsp T??D$gxx!!fTXXh%7%7f488Ir;R;R;V;VW_;`;`fdff 2r#ct|trd|DSt|tr d|DS|S)Ncni|]2\}}||tvrdnt|3S)z [REDACTED])rGSENSITIVE_KEYS safe_payloadrHrXrYs r!rJz safe_payload..Zs>rrrZ^Z[]^AGGII$?$?LL\RS__rrrr#c,g|]}t|SrK)ryrHxs r!r`z safe_payload..\s111A Q111r#)rrrsitemslistpayloads r!ryryXsa'4  srrbibobobqbqrrrr'4  2111111 Nr#clt|ts|Sd|DS)NcPi|]#\}}|tv ||$SrK)rGrxrzs r!rJz&secretless_payload..cs0 P P PTQ 0O0OAq0O0O0Or#)rrrsr~rs r!secretless_payloadr`s5 gt $ $ P PW]]__ P P PPr#c|dpidpidpd}|sdS tj|S#t$rYdSwxYw)NrequestpostDatatextrD)rFr)r* Exception)entryrs r!json_post_textrfs YYy ! ! 'R , ,Z 8 8 >B C CF K K QrD tz$ ttsA A)(A)cpt|didgS)Nlogentriesr.rFr,s r! har_entriesrps. T??  ub ) ) - -i < <z#find_best_request..s qtr#TkeyreverserR)rendswithrNrFr startswithappendrh) r preferredfallbackrrhostr-hmrscoreitems r!find_best_requestrzsxIH""+E22T4}}.//    )R0044Y?? @ @vvh   ''-2 , 4 4 SLE ??: & &  RKE w   RKE W $ $ RKEu~  "   T " " " " OOD ! ! ! !Ii^^TBBB1EaHH{1~ 4r#c|D]4}t|\}}}|dr ||kr|cS5t|zS)Nr)rrBASE_URL)r path_suffixrrrr-s r!find_first_urlrsX+E22T4 ==* + +  0C0CJJJ k !!r#clt|dpd}|dpg}t|dpd}|dkrd|vrdS|dkrdS|dkrdS|d ksd |vrd S|d krd S|d krd S|d krd Sd|vrdSd|vrdSd|vrdS|pdS)NrptTyperD queryDomains excelNameaccountrrrrrrrru 营销场景u计划u 关键词unknown)strrF)rrptqdexcels r!infer_report_keyrs gkk)$$* + +C ^ $ $ *B  K((.B / /E iGrMMw jz iy .E"9"9 g~~w jz f}}vw5zey  )r#cg}t}tt|D]z\}}t|\}}}|dr|dkr5t |}t |tsZt|} t|} | tj | ddf} | |vr| | | | t| |dp| |d| t!||d|d|d|d |d |d d ||d |S)NrrFT)r5 sort_keysrrr startTimeendTime splitType) report_key report_labelhar_entry_index endpoint_pathrpayload_previeworiginal_excel_namerrrrrcd|dtvr t|dndS)Nrc)REPORT_PRIORITYindexrs r!rz,collect_download_templates..s-1\?^mKmKm!6!6q!G!G!Gsur#)r)ri enumeraterrrrrrrsrrr)r;addr REPORT_LABELSrFrysort) har_path templatesseenidxrrrr-rr nonsecretrs r!collect_download_templatesrsI 55D H 5 566  U+E22T4}}.// 4;\3\3\  '''4((  %g.. &w// 4:ietTTTU $;;   $)--j'++k:R:R:`V`aa"> +G44#*;;{#;#;{{9--#KK77 [11{{9-- [11   NNuuNvvv r#ctt|dd\}}tj||d}|dkr0tj|dzddtjdz }n/tj||dzdtjdz }||fS)N-rR )days)mapintrSdtdate timedelta isoformat)monthymstartends r! month_rangers sEKKQ'' ( (DAq GAq!  EBwwga!eQ""R\q%9%9%99gaQ""R\q%9%9%99 ??  cmmoo --r#cvtjddt|d}|pdS)Nz[\\/:*?"<>|\s]+_alimama_report)resubrrT)rCs r!sanitize_filenamers6 6$c3t99 5 5 ; ;C @ @D  ###r#ct|}||dz }||dz }||dz }|r(|r|std|dt|}t |}t |}d|D}||d<||fS)N _cookie.json _headers.json _session.jsonzMISSING credentials for store=z; run extract-credentials firstci|] \}}||| SrKrKrzs r!rJz&load_secret_bundle..s#///$!QQ/A///r#rq)rexists SystemExitrur.r~) store secret_dir cookie_path headers_path session_pathrrMsessionrIs r!load_secret_bundlersj!!J%5555K5 7 7 77L5 7 7 77L     b|':':'<'<bLDWDWDYDYb`%```aaa k * *F %%G %%G//'--/////AAhK g:r#GETc |pi}d}|Otj|ddd}t|}|ddt j||||}t j||5}| }|j dd } | dd } tj | |j fcdddS#t$r&td |jd | d| ddwxYw#1swxYwYdS)NF),:)r5 separatorsr%z Content-Typezapplication/json;charset=UTF-8)r>rMmethodtimeoutrDr&r(znon-json response status=z ctype=z preview=)r)r;encoders setdefaultrrRequesturlopenreadrMrFdecoder*r RuntimeErrorstatus) rrrMrrr>reqrespbodyctypers r!open_json_requestrsmG Dz'*MMMTTU\]]w-->+KLLL . 4 P PC   W  5 5oyy{{   44{{79{55 o:d##T\1 oooooooo  o o om4;mmumm_cdhehdh_immnn n o oooooooooos%AD>$D  0D;;D>>EEcxt|trudD]E}||}t|tr|dr|cSF|D]}t |}|r|cSn/t|tr|D]}t |}|r|cSdS)N) downloadUrlrfileUrlossUrl)zhttp://zhttps://)rrrsrFrrvaluesextract_download_urlr)objrvalfounds r!rrs#t >  C''#,,C#s## 7N(O(O  ::<<  C(--E     C    C(--E    4r#dayc t|\}}t|d}||d<||d<t|D]H} | ds| ds| dkr|| dI||d<|d|d<|d |d <|d |d pd |d pd |d <t|d|d} | d| dddtj d|d<|dp tdz} |r d|d|dt|dSt!| d||d\} } | pidpid}| pidpid}|s+t#d|dd |d!t| ||d|d|d"S)#NrrrvsTypevsTime_sumrrrbizCode universalBPsourceasync_dowdloadrrrrDz %Y%m%d_%H%M%Sr create_urlrT)dry_runrrrPOST<)rrMrrr>taskIdinfookzcreate task failed for z: ok=z response=)r+rr response_ok)rrsrkeysrpoprFr rr&rdatetimenowstrftimerryrr)templaterMrrr( split_typerrrrlabelrr>rtask_idr-s r! create_taskr8!sU##JE38I&''G GKGIGLLNN#### >>( # # #s~~h'?'? #3&== KKT " " "%GK H--GH%kk.99GN y'++i"8"8"IMJJJ H--A1AGH   h|4h|6L M ME#llemmC&<&<llr{?P?P?Y?YZi?j?jllGK ++l # # U3T(TCTx /ET[\gTh~JKR~S~STT TFGW^`aaaGD!  ((.B33H==G JB  F # # )r . .t 4 4B vtXl5KttRTtt`lmq`r`rttuuuXl-CRYZeRfwy z zzr#Pc |dp tdz}tj|}|jdkr1|jdr|dd}ttj |j d }| t||d pd d tj|tj| }d} t!d|dzD]} t#|d|d\} } | } t%| } | r| | fcSd} | didid}n#t&$rd}YnwxYwt)t+jd|| |t/| ddt1j|t5d|d|dt7| )Nget_download_url+/commonapi/report/async/getDownloadUrl.jsonrz /commonapi/httpszbpcommon.alimama.com)schemerT)keep_blank_valuesr#r$)r+r#)queryrRrr)rrMrr>resultrpoll)eventr+attemptrrFr5ztask z# did not produce downloadUrl after z attempts; last=)rFCOMMONAPI_BASE_URLrrrrr-r_replacers parse_qslrAupdater urlunparse urlencoderangerrrrr)r;booltimesleep TimeoutErrorry)r7rMrattemptssecondsbaseparsedqsrlastrEr>rdlrs r!poll_download_urlrY?sO ;;) * * r/ADq/qD \ " "4 ( (F })))fk.D.D].S.S)8NOO fl$$V\T$JJ K KBIIW'++i2H2H2YMZZ[[[ , ! !&// 8N8Nr8R8R/"S"S T TC DHqL))  #CwPRSSSa !$ ' '  w;    XXfb))--h;;??IIFF   FFF  dj6W\bswxzs{s{||LQRRR S S S 7 ywyy8yyeqrvewewyy z zzs=F?? G Gci}|r |dr |d|d<tj||}tj|d5}|}dddn #1swxYwYt |}|jdd| |t|S)Nz User-AgentrLxrTr0) rFrrr r r rr8r9 write_bytesrj)rout_pathrMminimal_headersrrr>s r! download_filer_Xs O>7;;|,,>(/ (= % . o > >C   S  1 1Tyy{{H~~H O$666  t99s'BB B cdD]+} |||fcS#t$rY(wxYw|dddfS)N)z utf-8-siggb18030gbkr%r&rz utf-8-replace)r UnicodeDecodeError)rawencs r!decode_csv_bytesrfgsl. ::c??C' ' ' '!    D  ::gi: 0 0/ AAs  ,,c t|}|j}t||jd}t j|rHt j|5}|}||d<d|vri||dz }||kr'| | | t|ddd|cdddSd |D}|r| |d }t|\} } ||d z } | | d | t| | dd d dl} | } | j}d|_t'j| D]}||||dz }| || t|t j|ddnK#t0$r>}| t| ddt|dYd}~nd}~wwxYw|cdddS dddn #1swxYwY| t|ddd|S)zTReturn safe event details and produce a real .xlsx when platform returns zip-of-csv.)raw_file raw_bytes zip_namesz[Content_Types].xmlz.xlsxTxlsx)r valid_xlsxkindNc`g|]+}|d)|,S).csv)rGr)rHns r!r`z/normalize_download_artifact..s3HHHqQWWYY-?-?-G-GHHHHr#rror%r7zip_csv)csv_file csv_encodingrmr>)rrlxlsx_convertedF)rrlrt xlsx_errorrd)rr8rstatst_sizezipfile is_zipfileZipFilenamelistr\ read_bytesrJr rfr:openpyxlWorkbookactivetitlecsvreader splitlinesrsaver)raw_pathstemout_dirrBznames xlsx_path csv_names csv_bytesrrecsv_pathr}wbwsrowexcs r!normalize_download_artifactrpsvH~~HoGh--hmmoo6M N NF(## _X & & !JJLLE"'F; $--#nnn4 y(())(*=*=*?*?@@@ s9~~TSYZZ[[[        IHEHHHI FF9Q<00 ,Y77 c"]]]2##D7#;;; 3x==#W`aabbb A#OOO!**,,BB%BH"z$//*;*;<<'' # 'T... 8IGGI&&&MM3y>>I[\eIfIfz~""AAAA AAAMM3x==afvyz}v~v~""AAAAAAAAA9                        < MM3x==uMMNNN MsE)A:J0BJ2B9H,+J, I464I/*J/I44JJJc ttjdtjdt t|j gdddddS)NTr)scan-harextract-credentialscompare-credentialsbuild-downloads export-month)r-pythonrcommandsFr3r4) rr)r;rversionrSrrr expanduserrs r! cmd_doctorrs $*+##%%a($t//::<<==qqq  !  % % %&&&&&r#ct|j}t|j}i}|D]1}t|\}}}||ddz||<2t |jt |t|ddddt |d|Dd}|j rt|j |ttj |d d dS) NrrRc|dS)NrRrKrs r!rzcmd_scan_har..s rsturvr#TrcJg|] }d|D!S)c&i|]\}}|dk ||SrrKrzs r!rJz+cmd_scan_har...skO}O}O}Y]YZ\]nos|n|n|PQSTn|n|n|r#r~rHts r!r`z cmd_scan_har..syNQNQNQBCO}O}abahahajajO}O}O}NQNQNQr#)har entry_count hosts_topdownload_template_countdownload_templatesFr3r4)rrrrrFrrjrhr~rVr@rr)r;) r rrhostserrr-rBs r! cmd_scan_harrs\$(##G*4844I E --)!,, 4iia((1,d ]]3w<K|]}|VdSN) capitalize)rHrWs r! z*cmd_extract_credentials..s, L Lt!2!2 L L L L L Lr#OriginRefererz /index.htmlrrDTr0r#r$rrrr=z%/commonapi/report/async/findPage.jsonrStimespec) rsource_har_namebase_urlr#rrr'r< find_page_url cookie_names created_atrrq)r?rrz_qianniu_compare.json)r-r secret_files cookie_count has_csrfIdhas_loginPointIdqianniu_compareFr3r4)$rrrrrNrFrr~SAFE_HEADER_NAMESjoinrSr rrrr9rrhrZr/rCrrGrr1r2rr@compare_qianniu_dircompare_with_qianniurjrNsummarize_comparerr)r;)r rbestrr safe_headersrXrY canonicalrUrrrrcompare_summarypublics r!cmd_extract_credentialsrs^$(##G W % %D |GHHH DHHY++// :: ; ;BT""(bGL ((1 ! ! ! L Lqwws|| L L LLLI&'L #Hh///Ix-'?@@@FF8R((Mdo&&JTD111 JE-m<<AACCDDL>>.;;y)):]++h'' N33$W.OPP.1^^+.UU$koo''1191EE  Gzu2222X}4MVZ[[[[zu3333\$OOOOzu3333WTJJJJO _.}d>VWW:5 ? ? ??Y]^^^^!///E1H1H1HUJaJaJabL))7;;x0011 ^!.s7eeeQ eeer#)rercrdrarbr)comparerVrCrs @r!rrso t Cmmooee deeee.deeeD Jr#ct|j}t||j}t t jt|dddS)NFr3r4)ru cookie_filer qianniu_dirrr)r;r)r leftrs r!cmd_compare_credentialsrsR D, - -D"4)9::G $*&w//eA N N NOOOOOr#c t|j}t|jtjd|gdddgddgd}t|j|ttj d t|jt|d |Dd d d dS)NrSr)rrrrrrr) source_harrrrecommended_reportsoptional_reportsdeferred_reportsTcg|] }|d SrrKrs r!r`z'cmd_build_downloads..s8fJfJfJz{fghtfufJfJfJr#)r-rVtemplate_countreportsFr3r4) rrrrr1r2rr@rVrr)r;rj)r rmanifests r!cmd_build_downloadsrs*4844I$(mmkoo''1191EE???-w7'0 Htx""" $*DTX#i..fJfJ@IfJfJfJKKZ_hi j j jkkkkkr#cNt|}|dpgS)Nrrrts r! load_manifestr s$ T??D 88K &B&r#c t|j|j\}}t|j}d|jdDfd|D}|jr|d|j}|stdt|j }| ddg}|D]L}t||||j |j|j}|dd i|t#t%j|d d |jrqt)|d |||j|j\} } t/|d} || dz } t1| | |} t3| | }d|d|d | | d|}||t#t%j|d |dst7d|Nt9|dz |j|j |dt#t%jd|j|j t;|t=|dd ddS)Nc^g|]*}||+SrK)rTr|s r!r`z$cmd_export_month..s- F F FAAGGII Faggii F F Fr#rcBg|]}|dv|Srr])rHrwanteds r!r`z$cmd_export_month..s/FFFal(;(;v(E(E(E(E(Er#z!No matching reports in manifest: Tr0)r(r5rDr8FrFr+)rRrSrz.ziprL downloadedr)rDrr+ poll_attemptbytesrlz>&a oQXYaQbuALP]]T\] e djU333444yy&& ge^ceeff f gw//4:PTPZfl1m1mnnn $*D4: WZ[bWcWctwx~ttAAPU^_ ` ` `aaaaar#cttjdd}|dtt|dd}|d }|t |d }|d |d |t |d}|d |dd|d|t |d}|dd|dd|t |d}|d |d d|t |d}|ddd|dd|dd|dd|dd|dd |d!d"d"d#gd$%|d&t'|d(td)*|d+td,*|t |S)-Nzalimama-exportz.HAR-driven Alimama/Wanxiangtai report exporter)prog descriptionz --secret-dir)defaultcmdT)destrequireddoctor)funcrrz--outrz--store)rz--compare-qianniu-dirrz --cookie-filez --qianniu-dirrrz--monthzYYYY-MM)rhelpz --manifestz --reportszcampaign,scene,bidwordz --out-dirz --dry-run store_true)actionz --split-typerrzTday is default for exact natural-month aggregation; month follows the UI month split)rchoicesrz--limit)typez--poll-attemptsr9)rrz--poll-secondsr:)argparseArgumentParser add_argumentrSECRET_DIR_DEFAULTadd_subparsers add_parser set_defaultsrrrrrrr)prsps r! build_parserr/s%5CstttANN>3/A+B+BNCCC    5 5C  ! !BOOO$$$  # #BOOEOOGOOO&&& - . .BOOEOOIO---OO+,,,OO0O111 - . .BOOOdO333OOOdO333OO0O111 ) * *BOOEOOGdO+++OO,O---  ' 'BOOI9O===OOIO---OOL4O000OOK)AOBBBOOK$O///OOK O555OONEE7;KSiOjjjOOICO(((OO%CO<<<OO$3O:::OO)O*** Hr#cvt}||}||dSr)r parse_argsr)argvparserr s r!mainr[s1 ^^F   T " "DIIdOOOOOr#__main__)F)rNNr)Fr)r9r:r)=rr1rrhashlibr)r<rrrO urllib.parserurllib.requestrxpathlibrr rrGrxrrrr"r.r@rNrZroruryrrrrrrrrrrrrrr8rYr_rfrrrrrrrrrrrr__name__rKr#r!rs  TTUU $3mmmeeeaaa ! $   """PPPccc       QQQ ===!!! @"""6D...$$$   oooo$${{{{<{{{{2    BBB%%%P&&& < < <0<0<0