fKddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZmZmZmZddlmZmZmZmZmZddlmZddlmZddlmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'ddl(m)Z)dZ*dZ+dZ,dZ-e*e+e,gZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6ejne8Z9ddfddfdd fd!Z:e Gd"d#e Z;Gd$d%e<Z=Gd&d'e<Z>Gd(d)e<Z?dEd*Z@e3fd+ZAed,gd-ZBed.d/d0gZCGd1de'ej2ZEd3ZFd4eeEeGffd5ZHd6ZI dFd7eGd4eJfd8ZKd9ZLdGd:ZMGd;deEd?eGd4eJfd@ZQd?eGd4eeEfdAZRd4eGfdBZSdCeGd4eGfdDZTy)HN) namedtuple)Enumunique)AnyDictListOptionalTuple) atomic_helperdmiimporternet type_utils) user_data)util) write_json)Distro) EventScope EventType) launch_index)Paths)CloudInitPickleMixin)eventsdisabledlocalrpass FILESYSTEMNETWORK DataSourcez|EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.zredacted for non-root user cloud-name_unsetunknownz aws-chinac |dk(SNawscs r*?s 1:zaws-govc |dk(Sr$r&r's r)r*r*@s Q%Zr+z azure-chinac |dk(S)Nazurer&r's r)r*r*As qG|r+)zcn-zus-gov-chinac0eZdZdZdZdZdZdZdZde fdZ y ) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. cmdlineds system_cfgfallback initramfsreturnc|jSN)valueselfs r)__str__zNetworkConfigSource.__str__Rs zzr+N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGFALLBACK INITRAMFSstrr=r&r+r)r1r1Es. H BJHIr+r1ceZdZdZy)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesNr>r?r@rAr&r+r)rIrIVsOr+rIc eZdZy)DataSourceNotFoundExceptionNr>r?r@r&r+r)rLrLZr+rLceZdZdZy)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.NrJr&r+r)rPrP^sBr+rPcxtj|}g}g}|jD]\}}|r |dz|z}n|}|j|vs|j|vr|j |t |t r7|jdr&|j ||jdd||<t |tst|||} |j| jd|j| jd| ||<t||d<t||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:base64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancerG startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathrUmd_copyrT sens_keyskeyval sub_key_path return_vals r)r_r_bs/mmH%GINN$&S #c>C/LL IIK> )!!#~5   \ * c3 CNN9$=  & &| 4;;y"5GCL c4 2\>J & &z~~6K'L M   Z^^,<= >%GCL'&(&,,?%@G !" &y 1G  Nr+c|jdgs|Stj|}|jdD]M}|jd}|}|D])}||vst ||t s||dk7s%||}+|vsI|||<O|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. rUrR)getrVrWsplitr[r^)rc redact_valuererd path_partsobjpaths r)redact_sensitive_keysrss <<(" -mmH%GLL!12 %^^C(  D s4y$/JrN*$i   3;$CI % Nr+ URLParams)max_wait_secondstimeout_seconds num_retriessec_between_retriesDataSourceHostnamehostname is_defaultc zeZdZUeZdZdZdZdZdZ e je je je jfZee dfed<dZdZdZd Zej.ej2ej4ej6ej8hiZej.ej2hiZd efd efd ifd dddddf Z eee!e"fdfed<dZ#dZ$ee!dfed<dZ%dZ&e'e!ed<d Z(dIde)de*fdZ+de,ddfdZ-dZ.de/fdZ0de/fd Z1d!Z2d"Z3dJd#Z4de/fd$Z5dKd%Z6de/fd&Z7d'Z8dLd(Z9d)Z:d*Z;ed-Z?eZPdMd?ZQd@ZRdAeSefdBZTdAeSede/fdCZUdDZVeWdNdEZXe  ?sA== B,!B''B,c,tj|Sr9)robj_namer;s r)r=zDataSource.__str__as""4((r+cy)z#Check if running on this datasourceTr&r;s r) ds_detectzDataSource.ds_detectdr+c|jjtjk(rtj d|y|j j dg|jgk(rtj d|yy)aIOverride if either: - only a single datasource defined (nothing to fall back to) - commandline argument is used (ci.ds=OpenStack) Note: get_cmdline() is required for the general case - when ds-identify does not run, _something_ needs to detect the kernel command line definition. zOMachine is configured by the kernel commandline to run on single datasource %s.Tdatasource_listz5Machine is configured to run on single datasource %s.F)rrY parse_cmdlinerrrrmr;s r)override_ds_detectzDataSource.override_ds_detecthsv ;;   -/"7"7"9 9 II(   \\  / 4 E IIG r+c|jr|jS|jr&tj d||jStj d|y)z&Overrides runtime datasource detectionz8Detected platform: %s. Checking for active instance dataz#Datasource type %s is not detected.F)r _get_datarrrr;s r)_check_and_get_datazDataSource._check_and_get_datasX  " " $>># # ^^  IIJ >># # II;T Br+c|jj}|j}|j}|d}diddgd|d|dt |j |j |jd|j d |j d |d d d |d dd|d dd|jd|jd|dd|d|d|ddd|d||dd|j |j|d|ddiS)z2Return a dictionary of standardized metadata keys.sys_infov1 _beta_keys subplatformavailability-zoneavailability_zonecloud_idr cloud_namerdistrdistro_versionrdistro_releaseplatformpublic_ssh_keyspython_versionpython instance-id instance_idkernel_releaseunamelocal-hostnamelocal_hostnamevariant)machineregionrsystem_platformr) get_hostnamerzget_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r< instance_datarrrsysinfos r)_get_standardized_metadataz%DataSource._get_standardized_metadatas**,55**,  22 + }o#%6$%6.OOT[[$2D2D doodoo'&/!,!'&/!"4!'&/!"4D.."4#;#;#=!'("3{ {!"!''"21"5#$!.%&!.'(#7+A.++#//#*:#6"9-1  r+c|jsy|r|}n |j}|D]\}}t||st|||!|sd|_yy)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cacherrsetattr)r< attr_defaults attr_values attributer:s r)clear_cached_attrszDataSource.clear_cached_attrss_    'K33K + 0 IutY'i/ 0 %D r+c\d|_|j}|s|S|j|S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r< return_values r)get_datazDataSource.get_datas5 !//1   ""$r+c|rXtjj|jjr%t ||jj dt|drHtjt|d}|jdd|jddd|i}nbdd|jii}t|drt|d}|tk7r||dd<t|d rt|d }|tk7r||dd <t|dd <tj|j|d <d |d d <tj|d |d <d|d d <t!j"|d<|j%|j'| t)j*|}t-t/j0||j2}|jj?d} |djAdd} tjjC|jjDd} t!jF| d| | dd} | d| } tjjI| rtjjK| } t!jL| | d| r| | k7rt!jN| tQ| |d|jj?d}tQ|tS|y#t4$r)}t6j9dt;|Yd}~yd}~wt<$r)}t6j9dt;|Yd}~yd}~wwxYw)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. obj_pkl_crawled_metadatarNrr3 meta_datarr_docrz TZZ%7%7D  =/8*5(2G!,oQxj9 77>>- (!#!1!1-!@  'dC "48I"I MM, -&UCJJ**?; 93NCD1  KKA3q6 J!  KKA3q6 J s$"?L M?&M  M?M::M?ctd)z@Walk metadata sources, process crawled data and save attributes.zlSubclasses of DataSource must implement _get_data which sets self.metadata, vendordata_raw and userdata_raw.)NotImplementedErrorr;s r)rzDataSource._get_data$s! D  r+c R|j} t|jjd|j}|j} tdt|jjd|j}|j} t|jjd|j}|j} t|jjd|j}t||||S#t$r9t j td|jjd|Y wxYw#t$rE|j}t j td|jjd|YwxYw#t$r9t j td|jjd|Y!wxYw#t$r9t j td |jjd|Y*wxYw) zReturn the Datasource's preferred url_read parameters. Subclasses may override url_max_wait, url_timeout, url_retries. @return: A URLParams object with max_wait_seconds, timeout_seconds, num_retries. max_waitz6Config max_wait '%s' is not an int, using default '%s'rtimeoutz5Config timeout '%s' is not an int, using default '%s'retriesz5Config retries '%s' is not an int, using default '%s'rxzAConfig sec_between_retries '%s' is not an int, using default '%s') url_max_waitintrrm ValueErrorrlogexcr url_timeoutmax url_retries Exceptionurl_sec_between_retriesrt)r<rrrrxs r)get_url_paramszDataSource.get_url_params+s$$ 4;;??:t7H7HIJH"" !SDE  E A FF>G! G!$>H&%H&c|j.|jj|j|_|r|j |jS|jSr9)rrprocessget_userdata_raw _filter_xdata)r< apply_filters r) get_userdatazDataSource.get_userdatagsM ==  LL001F1F1HIDM %%dmm4 4}}r+c|j.|jj|j|_|jSr9)rrrget_vendordata_rawr;s r)get_vendordatazDataSource.get_vendordatans5 ?? ""ll2243J3J3LMDOr+c|j.|jj|j|_|jSr9)rrrget_vendordata2_rawr;s r)get_vendordata2zDataSource.get_vendordata2ss:    ##||33D4L4L4NOD r+ct|ds|jj|_|js|jj|_|jS)N_platform_type)rrrYr+r;s r)rzDataSource.platform_typexsLt-."&++"3"3"5D """&++"3"3"5D """r+ct|ds|j|_|js|j|_|jS)aReturn a string representing subplatform details for the datasource. This should be guidance for where the metadata is sourced. Examples of this on different clouds: ec2: metadata (http://169.254.169.254) openstack: configdrive (/dev/path) openstack: metadata (http://169.254.169.254) nocloud: seed-dir (/seed/dir/path) lxd: nocloud (/seed/dir/path) _subplatform)r_get_subplatformr-r;s r)rzDataSource.subplatformsEt^, $ 5 5 7D   $ 5 5 7D    r+cDt|drdt|dzStS)z?Subclasses should implement to return a "slug (detail)" string.metadata_addressz metadata (%s))rrMETADATA_UNKNOWNr;s r)r.zDataSource._get_subplatforms& 4+ ,"WT3E%FF Fr+c,|jr |jS|jr|jjtr|jjt}t |t r!|j |_|jS|jj |_tjdtt||jS|jj |_|jS)zReturn lowercase cloud name as determined by the datasource. Datasource can determine or define its own cloud product name in metadata. z5Ignoring metadata provided key %s: non-string type %s) _cloud_namercrmMETADATA_CLOUD_NAME_KEYr[rGrY_get_cloud_namerrtype)r<rs r)rzDataSource.cloud_names   ## # ==T]]../FG**+BCJ*c*#-#3#3#5 $(#7#7#9#?#?#A  K+$ $335;;=D r+c|jS)zReturn the datasource name as it frequently matches cloud name. Should be overridden in subclasses which can run on multiple cloud names, such as DatasourceEc2. )rr;s r)r5zDataSource._get_cloud_names {{r+cX|jsyd|jvr|jdSy)Nz launch-index)rcr;s r)rzDataSource.launch_indexs*}} T]] *==0 0r+ctjtj|jg}|}|D]}|j |}|Sr9)rFilterrsafe_intapply)r< processed_udfiltersnew_udfs r)r!zDataSource._filter_xdatasN    d.?.? @ A  %AWWV_F % r+cyNFr&r;s r)is_disconnectedzDataSource.is_disconnectedr+c|jSr9)rr;s r)r zDataSource.get_userdata_raws   r+c|jSr9)rr;s r)r%zDataSource.get_vendordata_raws"""r+c|jSr9)rr;s r)r(zDataSource.get_vendordata2_raws###r+ciSr9r&r;s r)get_config_objzDataSource.get_config_objs r+cJt|jjdS)Nz public-keys)normalize_pubkey_datarcrmr;s r)rzDataSource.get_public_ssh_keyss$T]]%6%6}%EFFr+cy)a5Publish the public SSH host keys (found in /etc/ssh/*.pub). @param hostkeys: List of host key tuples (key_type, key_value), where key_type is the first field in the public key file (e.g. 'ssh-rsa') and key_value is the key itself (e.g. 'AAAAB3NzaC1y...'). Nr&)r<hostkeyss r)publish_host_keyszDataSource.publish_host_keyssr+cddi}|jD]V\}}|j|s|D]:}d||t|d}tjj |s6|ccSXy)Nsd)vdxvdvtbz/dev/)rXr\lenrrrr)r< short_namemappingsnfromtlistntocands r) _remap_devicezDataSource._remap_devicesx ./&nn. NUE((/ &):c%jl+CD77>>$'K  r+cyr9r&)r<_names r)device_name_to_devicez DataSource.device_name_to_devices r+ct|j} |jj}|S#t$rY|SwxYw)z, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. localdomain localhostFrNT.rzip-%srr)rcrmryrrget_fqdn_from_hostsfindrGrnris_ipv4_address gethostbyaddrr]rTr) r<fqdn resolve_ip metadata_only defdomaindefhostdomainr{toksrz hosts_fqdnlhosts r)rzDataSource.get_hostnames&"  }}DMM$5$56F$G)$ ;; D((*H;&! 11(;Jjooc2Q6:,,S1hmmC0148}**3/ ),+MM"23E""5)--e4Dt9??3/D#emmC&==>D{{3' t9q=AwHXXd12h'FAwH Fi'"*F3H!(J77r+c:|jj|S)N) data_source)rget_package_mirror_infor;s r)r|z"DataSource.get_package_mirror_infofs{{22t2DDr+source_event_typesci}|D][}|jjD]<\}}||vs |j|s t||<||j |>]|Sr9)supported_update_eventsrXrmsetadd)r<r}supported_eventsevent update_scope update_eventss r)get_supported_eventszDataSource.get_supported_eventsis~24' >E--335 >M)+// =9<(6$\266u= > > r+c |j|}|jD]h\}}tjd|jdj |Dcgc]}|jc}|j d|ztffj|r#|j |j}|rytjd|dj |Dcgc]}|jc}ycc}wcc}w)aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %s, z _%s_configTz(Datasource %s not updated for events: %sF) rrXrrr:rrrr)r<r}rscopematched_eventsrresults r)update_metadata_if_supportedz'DataSource.update_metadata_if_supportedvs 445GH%5%;%;%= F !E> IIL  NC55;;CD   # #lU&:E%B$D E F   # # %]]_F 6  II0BCuu{{C D !DDs C1C6cyrBr&)r<rs r)check_instance_idzDataSource.check_instance_idrDr+c~|t}|t}|D]'}|||vr|cStjd|||cS|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESrr) candidatesdefaultvalid candidates r)_determine_dsmodezDataSource._determine_dsmodes_ ?$G =!E# I E!   ;Y r+cyr9r&r;s r)network_configzDataSource.network_configrr+cy)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr&)r<is_new_instances r)setupzDataSource.setups r+cy)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr&)r<cfgrs r)activatezDataSource.activates r+r9)r&T)F)FFF)NN)\r>r?r@rdsmoder`rr3r+r-r1rBrFrDrCr~r __annotations__rrrrrrrBOOT_NEW_INSTANCEBOOT BOOT_LEGACYHOTPLUGrdefault_update_eventsrrrGrrrrrr _ci_pkl_versionrrrrrr=boolrrrrrrrrrr#r&r)propertyrrr.rr5rr!rCr r%r(rIrrNr[r^rarrrrr|rrrr staticmethodrrrrr&r+r)rrs* F"NFKN L $$%%&& ?E"5s":;LKK@   ' ' NN  ! !        ' '    R ! 9%c3h 45 L 0U38_ " /3hsm2O##u#.1114)4D. $ L&& $ Pd 4 :Jx  ##!!$   .!#$ G  KK++1 F8PE  tI  $"&y/$ $L*  r+) metaclasscng}|s|St|tr|jSt|ttfr t|St|t r]|j D]J\}}t|tr|g}t|ttfs0|D]}|s|j|L|Sr9)r[rG splitlineslistrr^rXrZ) pubkey_datakeys_keynameklistpkeys r)rKrKs D  +s#%%''+c{+K  +&!,!2!2!4 * Xu%%%$-!*D D) * * Kr+r7c t|||}|Dcgc]}tj|} }t|vrdnd} tj d| | t | |D]\} } tjd| jddzd| d| d | d | | } | 5tj d | | |||}|jtjgr/d | d| | _ |tj| fcdddcS dddddj#| z}t%|cc}w#1swY/xYw#t$rtj td| YwxYw)Nnetworkrz#Searching for %s data source in: %sz search-%srrSzsearching for z data from zno z data found from )name descriptionmessageparentz%Seeing if we can get any data from %szfound zGetting data from %s failedz4Did not find any data source, searched classes: (%s)r) list_sourcesrr DEP_NETWORKrrziprReportEventStackr]rrrrrrrrrL)rrrds_depscfg_listpkg_listreporterds_listr@ds_namesrrclsmyrepsmsgs r) find_sourcersz8Wh7G0781 ##A&8H8#w.9GDII3T8D7+A c''t||L"==;?F26=   A 9 A3G/11001@DT$JEMz22378 9 9 9A& A499D C &c **59 9 9 A KK:C @ As6D:E  A$D? E E ?E E  $E32E3cbg}tjd||||D]}tj|}tj||dg\}}|stj d||D]@}tj |} t| d} | |} | s/|j| |S)zReturn a list of classes that have the same depends as 'depends' iterate through cfg_list, loading "DataSource*" modules and calling their "get_datasource_list". Return an ordered list of classes that match (if any) zLLooking for data source in: %s, via packages %s that matches dependencies %sget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) rrr "match_case_insensitive_module_name find_moduleerror import_modulerr`) rdependsrsrc_listr3ds_namem_locs _looked_locsm_locmodlistermatchess r)rrs HII 8 ==bA'33 X 56   II$   E((/CS"78FWoG(  $ Or+fieldcz|sytj|}|sy|j|jk(SrB)r read_dmi_datarY)rr dmi_values r)instance_id_matches_system_uuidr6s; !!%(I     )//"3 33r+c|st}|st}|tk(r |tk7r|S|StjD](\}}|\}}|j|s||s&|cS|tk7r|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)r1CLOUD_ID_REGION_PREFIX_MAPrXr\)rrrprefix cloud_id_testr valid_clouds r)rrDs % ! !! ) ) !;!A!A!C "/;   V $Z)@O%% Or+c|syt|tr|St|trtj|St|t r+|durt |jddStdtdt|z)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) r[rGrrVrWr^convert_vendordatarmrr6)datars r)rrWsz $ $}}T""$ d?%dhh|&  C;UCs 2A$AA$BBcdd} tj|}|sy tj|S#t$r@}tjj |rt jd||Yd}~]d}~wwxYw#t$rYyt$rtjt d|YywxYw)zBUse pickle to deserialize a instance Datasource from a cache file.Nzfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rload_binary_filerrrrisfilerrrrrIr)rpickle_contentsrs r)pkl_loadrsOE//6 ||O,, E 77>>% KK95! DE +  C>Fs+2A> A;6A66A;> B/ #B/.B/c<ttjS)zCheck if command line argument for this datasource was passed Passing by command line overrides runtime datasource detection )parse_cmdline_or_dmir get_cmdliner&r+r)rrs  0 0 2 33r+inputcvtjd|}tjd|}tjd|}|xs|xs|}|xs|}|rA|jdj}t j d|d|dd|d  |r"|jdr|jdSy ) Nz(?:^|\s)ds=([^\s;]+)z(?:^|\s)ci\.ds=([^\s;]+)z (?:^|\s)ci\.datasource=([^\s;]+)rz7Defining the datasource on the commandline using ci.ds=z or ci.datasource=z23.2zUse ds=z instead) deprecateddeprecated_version extra_messagerS)researchgroupstripr deprecate)r ds_parse_0 ds_parse_1 ds_parse_2r3rrs r)rrs2E:J6>J>FJ  /z /ZB)zJ!!!$**, !!!'* &#F884  bhhqkxx{ r+)rSr&)z system-uuidr)UabcrVrloggingrrr collectionsrenumrrtypingrrrr r cloudinitr r r rrrrrcloudinit.atomic_helperrcloudinit.distrosrcloudinit.eventrrcloudinit.filtersrcloudinit.helpersrcloudinit.persistencercloudinit.reportingrDSMODE_DISABLED DSMODE_LOCALr DSMODE_PASSrDEP_FILESYSTEMr DS_PREFIXrREDACT_SENSITIVE_VALUEr4rr1 getLoggerr>rrr1rrIrLrPr_rsrtryABCMetarrKrGrrrrrrIOErrorrrrrrrr&r+r)r s  "33CC%.$1*#6&   ,?   < 6'g! - ./03 4 $   PiP ) CyC F2H:     _ %_ D:+ :s?+B!J, 4 4  4&J0 W :cd$CHZ0*4s4r+