a d9 @sfddlmZmZddlmZddlmZddlZddlZddlm Z e e ddd Z Gd d d Z dS) )OptionalDict) EinopsError)ParsedExpressionN)_productaxesZreport_messagecCst|dkrt||dS)Nr)lenrformatrr ^/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/einops/layers/_einmix.py _report_axes s rcsReZdZd fdd ZeeeeeeeedddZddZd d Z Z S) _EinmixMixinNc sft||_||_||_|_|d\}}t|}t|t|}tt j h|j |j d|j szj sz|j rt dtdd||fDrt dd|vsd|vrt d |d } d } d } td d|jDr*g|jD] } | 7qd } |d| } fd d|jD} tddjDrrgjD]} | 7qJd } | d|} || | | i|j D]}|vrt d|qtt t h|j |j dtt |j h|j j dt|j dkrtdfdd|jD}tfdd|jD}|d urt|tsPt dt|}tt |j j dtt |j t dg}jD]8}|D],}||j vr||n |dqqnd }d }d|d}d|d}|||||h|j j |j ddttjDtdfd d! }d"|| || | |_!d S)#a EinMix - Einstein summation with automated tensor management and axis packing/unpacking. EinMix is an advanced tool, helpful tutorial: https://github.com/arogozhnikov/einops/blob/master/docs/3-einmix-layer.ipynb Imagine taking einsum with two arguments, one of each input, and one - tensor with weights >>> einsum('time batch channel_in, channel_in channel_out -> time batch channel_out', input, weight) This layer manages weights for you, syntax highlights separate role of weight matrix >>> EinMix('time batch channel_in -> time batch channel_out', weight_shape='channel_in channel_out') But otherwise it is the same einsum under the hood. Simple linear layer with bias term (you have one like that in your framework) >>> EinMix('t b cin -> t b cout', weight_shape='cin cout', bias_shape='cout', cin=10, cout=20) There is restriction to mix the last axis. Let's mix along height >>> EinMix('h w c-> hout w c', weight_shape='h hout', bias_shape='hout', h=32, hout=32) Channel-wise multiplication (like one used in normalizations) >>> EinMix('t b c -> t b c', weight_shape='c', c=128) Separate dense layer within each head, no connection between different heads >>> EinMix('t b (head cin) -> t b (head cout)', weight_shape='head cin cout', ...) ... ah yes, you need to specify all dimensions of weight shape/bias shape in parameters. Use cases: - when channel dimension is not last, use EinMix, not transposition - patch/segment embeddings - when need only within-group connections to reduce number of weights and computations - perfect as a part of sequential models - next-gen MLPs (follow tutorial to learn more) Uniform He initialization is applied to weight tensor and encounters for number of elements mixed. Parameters :param pattern: transformation pattern, left side - dimensions of input, right side - dimensions of output :param weight_shape: axes of weight. Tensor od this shape is created, stored, and optimized in a layer :param bias_shape: axes of bias added to output. :param axes_lengths: dimensions of weight tensor z->z7Unrecognized identifiers on the right side of EinMix {}z/Ellipsis is not supported in EinMix (right now)css|] }|jVqdSN)Zhas_non_unitary_anonymous_axes).0xr r r Iz(_EinmixMixin.__init__..z2Anonymous axes (numbers) are not allowed in EinMix()z,Parenthesis is not allowed in weight shape: Ncss|]}t|dkVqdSNr rgroupr r r rQr csi|]\}}|vr||qSr r )rnamelength)namesr r Wrz)_EinmixMixin.__init__..css|]}t|dkVqdSrrrr r r rYrz*Dimension {} of weight should be specifiedzAxes {} are not used in patternzWeight axes {} are redundantrzCEinMix: weight has no dimensions (means multiplication by a number)csg|]\}|qSr r raxis) axes_lengthsr r prz)_EinmixMixin.__init__..cs g|]\}|jvr|qSr ) identifiersr!)r#rightr r r$rrzAbias shape should be string specifying which axes bias depends onz"Bias axes {} not present in outputz#Sizes not provided for bias axes {}rg?cSsi|]\}}||qSr r )rletterkr r r r rr csdfdd|DS)Nc3s|]}|VqdSrr r!mapping2lettersr r rrz<_EinmixMixin.__init__..write_flat..)joinr*r,r r write_flatsz)_EinmixMixin.__init__..write_flatz {},{}->{})"super__init__pattern weight_shape bias_shaper#splitrrset differencer%Z has_ellipsisrany compositionr.items_create_rearrange_layersr r warningswarnr isinstancestrappend_create_parameterszipstringascii_lowercaselistZflat_axes_orderZeinsum_pattern)selfr2r3r4r#Z left_patternZ right_patternleftZweightpre_reshape_patternpre_reshape_lengthspost_reshape_patternrr9r"Z _weight_shapeZ_fan_inZbiasZ _bias_shaper Z_bias_input_size weight_bound bias_boundr/ __class__)r#r-rr&r r1s(                   z_EinmixMixin.__init__)rHrIrJpost_reshape_lengthscCs tddS)N.Should be defined in framework implementationsNotImplementedError)rFrHrIrJrOr r r r;sz%_EinmixMixin._create_rearrange_layerscCs tddS)z Shape and implementations rPNrQ)rFr3rKr4rLr r r rAsz_EinmixMixin._create_parameterscCslt|j}|d|jd7}|jdur8|d|jd7}|jD]\}}|d||7}qBd|jj|S)Nz, ''z, {}={}z{}({})) reprr2r3r4r#r:r rN__name__)rFparamsr"rr r r __repr__s  z_EinmixMixin.__repr__)N) rU __module__ __qualname__r1rr?rr;rArW __classcell__r r rMr rs r)typingrrZeinopsrZeinops.parsingrr<rCrr6r?rrr r r r s