a dD @s.ddlmZddlZddlmZddlmZddlZddl m Z ddl m Z m Z mZmZddlmZmZmZmZzt nt }t |j |j |j|j|j|||d}|S)N)context_lengthwidthr layers ls_init_valueZ output_dim act_layer norm_layer) isinstancedictrr rZGELUtorchfloat16Zbfloat16rrr r-r.r r/r0) embed_dimmultimodal_cfgr+r,r1r2decoderr)r)r*_build_text_decoder_tower5s r:cseZdZd!eeeeeej e dfdd Z ej j d"dd Zd#d d Zd$d d Zd%ddZd&ddZd'ddZd(ddZd)dd ZZS)*CoCaFNr)r8text_cfg vision_cfgr+r,pad_idc stt|tr"tfi|n|}t|tr>tfi|n|}t|trZtfi|n|}t||||d|_t |dr|j dur|j n|j }t ||||d|_ t||||d|_ttgtd|_||_dS)N)r7r<r+r, hf_model_name)r7r=r+r,)r8r+r,g$I$I,@)super__init__r3r4rr r rtexthasattrr? vocab_sizer visualr: text_decoderr Parameterr5onesnplog logit_scaler>) selfr7r8r<r=r+r,r>rD __class__r)r*rAPs<  z CoCa.__init__TcCs(|j||j||j|dS)N)rEset_grad_checkpointingrBrF)rLenabler)r)r*rO}s  zCoCa.set_grad_checkpointingcCs,||\}}|r tj|ddn|}||fSNdim)rEF normalize)rLimagesrV image_latentZ tokens_embsr)r)r* _encode_imageszCoCa._encode_imagecCsH|r|ddddfn|}||\}}|rt||}n>|d krVt||}n&tdddttdd||\}}|durtj |j d df|tjd | }|j}t|j }|dkr|dddf}|j d}||}|dd| df}|j d}|||||ddddddf}|dddf| k|dddf| kB}tj |j d df|tjd | }|r|s@qnn||ddf}|||ddf|} |||ddf| } tj| |dd }!|d|kr&tj t|df|tjd | ||ddf<nt|!d||ddf<tj ||fdd }|d7}||drqnq|dkr|d }|||WdS1s0YdS)NzSPlease install transformers for generate functionality. `pip install transformers`.z'seq_len must be larger than min_seq_lenii) max_lengthr) image_inputs pad_token_id eos_token_id sot_token_id num_beamsnum_beam_groups min_seq_lenstopping_criterialogit_processorrrdevicedtyperSrrz!generation_type has to be one of z| z | z |.F)rXrerZr`rR) _has_transformersr5Zno_gradr>rrrrrry_generate_beamsearchrbcatrHrzGENERATION_TYPES ValueErrorjoinlistkeysrYlongZtraininglenevalallrUZsoftmaxsumZ multinomialZsqueezeZtrain)"rLrdrBZseq_lenZ max_seq_lenZ temperatureZgeneration_typerrrprqrrrsrtrurvZrepetition_penaltyZfixed_output_lengthrwryoutput logit_warperrXreZ was_trainingZnum_dimscur_lenoutxr`masksampleZfiltered_logitsZprobsr)r)r*generates     ,        "(  0    z CoCa.generatec - s|j} |jd} tj|dd}||\} }tj| df| tjd}||}t| | |d}| durztt ||dgn| }t |j } |j |j }|}|j\}}d}| |krtd| d|d tj| fd tj| d }d|dddd|f<|| f}tj| |j| d }tj| tj| d }t||d }||d |dd| |d}t|D]}||}t||}||}g}t| D]&|fddt||Dq||} |d|dddf}!|!jd}"|| |!||d}#|#||d}$|$|#}$|$| ||"}$tj|$d|dddd\}$}%tj|%|"dd}&|%|"}%|durlt|dnd}'|j| |$|%|&|||'d}(|(d||<|(d})|(d}*| |*||<tj| |*ddf|)dgdd} | dddf||<tj|*|dd||*|||<qhtj||dgdd}|d}|jsL||drqLq|dur`t|dnd}+|j |||%|&|||j!|+d },|,d!S)"NrrSrrx) batch_sizersryrt)rqz)Batch dimension of `input_ids` should be z , but is r{ge)rzry) input_idsrorWrBF)rZrXrecsg|]}|qSr)r)).0idxZ batch_idxrsr)r* qz-CoCa._generate_beamsearch..r`rR)current_tokensbeam_group_idxT)rTZlargestsortedfloor)Z rounding_moder))rprq beam_indicesZnext_beam_scoresZnext_beam_tokensZnext_beam_indices)rprqrnr sequences)"ryrbr5Zrepeat_interleaverYrHrrrrrZ _beam_hypsrsrtrfullfloatviewzerosrzprepare_inputs_for_generationrangeminextend unsqueezeZ expand_asZtopkdivrprocessr~Zis_donefinalizern)-rLrorprqrrrsrtrurvrwrryrrXrerZ beam_scorerZlogits_processorZ num_sub_beamsZbatch_beam_sizerrZ beam_scoresrZreordering_indicesZ model_inputsoutputsrZgroup_start_idxZ group_end_idxZ group_sizeZbatch_group_indicesZgroup_input_idsZnext_token_logitsrDZnext_token_scores_processedZnext_token_scoresZ next_tokensZ next_indicesZprocess_beam_indicesZ beam_outputsZbeam_next_tokensZbeam_idxZfinal_beam_indicesZsequence_outputsr)rr*r}"s             $  zCoCa._generate_beamsearch)FNr)T)T)TT)T)TT)TNN)NrgrhrirrjrNNNrkrlrmNriF) NNNrkrlrmNNN)r$r%r&rr r boolrr5rzr'rAZjitignorerOrYr\r^r_rfrr} __classcell__r)r)rMr*r;OsZ-       ~r;cKsz|r|dddfd}|dd}|dd}|durf|durf|dd}||dkdnd}|||||dS)NrRattention_mask position_idsrr)rBrWZpast_key_valuesrr)rgetrZcumsumZ masked_fill_)rroZpastkwargsrrr)r)r*rs  r)FN)N))typingrr5rZtorch.nnrrUnumpyrIZ dataclassesrZ transformerrrr r modelr r r rZ transformersrrrrrrrrrr| ImportErrorerrrzr:Moduler;rr)r)r)r*s@    (   j