dsmith111/phi3:mini-qnn-exported/ortgenai

dsmith111/ phi3:mini-qnn-exported

30 Downloads Updated 7 months ago

ortgenai

53e87acd30e0 · 249kB

onnxruntime-genai:��

�

past_key_values.0.key:/model/attn_mask_reformat/past_key_subgraph/Shape/output_01/model/attn_mask_reformat/past_key_subgraph/Shape"Shape

w/model/constants/INT64/2/model/constant_nodes/INT64/2"Constant*2

value*&B/model/constants/INT64/2J��

�

:/model/attn_mask_reformat/past_key_subgraph/Shape/output_0

/model/constants/INT64/2;/model/attn_mask_reformat/past_key_subgraph/Gather/output_02/model/attn_mask_reformat/past_key_subgraph/Gather"Gather*

axis��

w/model/constants/INT64/1/model/constant_nodes/INT64/1"Constant*2

value*&B/model/constants/INT64/1J��

�

input_ids=/model/attn_mask_reformat/input_ids_subgraph/Shape_2/output_04/model/attn_mask_reformat/input_ids_subgraph/Shape_2"Shape

�

=/model/attn_mask_reformat/input_ids_subgraph/Shape_2/output_0

/model/constants/INT64/1>/model/attn_mask_reformat/input_ids_subgraph/Gather_2/output_05/model/attn_mask_reformat/input_ids_subgraph/Gather_2"Gather*

axis��

�

>/model/attn_mask_reformat/input_ids_subgraph/Gather_2/output_0

;/model/attn_mask_reformat/past_key_subgraph/Gather/output_0;/model/attn_mask_reformat/input_ids_subgraph/Add_1/output_02/model/attn_mask_reformat/input_ids_subgraph/Add_1"Add

/model/constants/INT64/[0]/model/constant_nodes/INT64/[0]"Constant*6

value**B/model/constants/INT64/[0]J��

�

;/model/attn_mask_reformat/input_ids_subgraph/Add_1/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_3/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_3" Unsqueeze

�

>/model/attn_mask_reformat/input_ids_subgraph/Gather_2/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_4/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_4" Unsqueeze

�

>/model/attn_mask_reformat/input_ids_subgraph/Gather_2/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_5/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_5" Unsqueeze

�

>/model/attn_mask_reformat/input_ids_subgraph/Gather_2/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_6/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_6" Unsqueeze

�

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_4/output_0

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_5/output_0>/model/attn_mask_reformat/input_ids_subgraph/Concat_2/output_05/model/attn_mask_reformat/input_ids_subgraph/Concat_2"Concat*

axis��

�

>/model/attn_mask_reformat/input_ids_subgraph/Concat_2/output_0G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_2/output_0>/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_2"ConstantOfShape*3

value*'

Bmake_input_ids_subgraph_shapeJ��

�

G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_2/output_0=/model/attn_mask_reformat/input_ids_subgraph/Shape_4/output_04/model/attn_mask_reformat/input_ids_subgraph/Shape_4"Shape

�/model/constants/INT64/[-1] /model/constant_nodes/INT64/[-1]"Constant*7

value*+B/model/constants/INT64/[-1]J��

�,/model/constants/INT64/[9223372036854775807]1/model/constant_nodes/INT64/[9223372036854775807]"Constant*H

value*<B,/model/constants/INT64/[9223372036854775807]J��

�

=/model/attn_mask_reformat/input_ids_subgraph/Shape_4/output_0

/model/constants/INT64/[-1]

,/model/constants/INT64/[9223372036854775807]

/model/constants/INT64/[0]=/model/attn_mask_reformat/input_ids_subgraph/Slice_1/output_04/model/attn_mask_reformat/input_ids_subgraph/Slice_1"Slice

�

=/model/attn_mask_reformat/input_ids_subgraph/Slice_1/output_0

/model/constants/INT64/[0]?/model/attn_mask_reformat/input_ids_subgraph/Squeeze_1/output_06/model/attn_mask_reformat/input_ids_subgraph/Squeeze_1"Squeeze

�

?/model/attn_mask_reformat/input_ids_subgraph/Squeeze_1/output_0

/model/constants/INT64/[0]>/model/attn_mask_reformat/input_ids_subgraph/output_0/output_05/model/attn_mask_reformat/input_ids_subgraph/output_0" Unsqueeze

/model/constants/INT64/[1]/model/constant_nodes/INT64/[1]"Constant*6

value**B/model/constants/INT64/[1]J��

�

>/model/attn_mask_reformat/input_ids_subgraph/output_0/output_0

/model/constants/INT64/[1]>/model/attn_mask_reformat/input_ids_subgraph/Concat_3/output_05/model/attn_mask_reformat/input_ids_subgraph/Concat_3"Concat*

axis��

�

G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_2/output_0=/model/attn_mask_reformat/input_ids_subgraph/Shape_5/output_04/model/attn_mask_reformat/input_ids_subgraph/Shape_5"Shape

�

=/model/attn_mask_reformat/input_ids_subgraph/Shape_5/output_0

/model/constants/INT64/[-1]

,/model/constants/INT64/[9223372036854775807]

/model/constants/INT64/[0]=/model/attn_mask_reformat/input_ids_subgraph/Slice_2/output_04/model/attn_mask_reformat/input_ids_subgraph/Slice_2"Slice

�

=/model/attn_mask_reformat/input_ids_subgraph/Slice_2/output_0

/model/constants/INT64/[0]?/model/attn_mask_reformat/input_ids_subgraph/Squeeze_2/output_06/model/attn_mask_reformat/input_ids_subgraph/Squeeze_2"Squeeze

w/model/constants/INT64/0/model/constant_nodes/INT64/0"Constant*2

value*&B/model/constants/INT64/0J��

�

/model/constants/INT64/0

?/model/attn_mask_reformat/input_ids_subgraph/Squeeze_2/output_0

/model/constants/INT64/1;/model/attn_mask_reformat/input_ids_subgraph/Range/output_02/model/attn_mask_reformat/input_ids_subgraph/Range"Range

�

;/model/attn_mask_reformat/input_ids_subgraph/Range/output_0

/model/constants/INT64/1;/model/attn_mask_reformat/input_ids_subgraph/Add_2/output_02/model/attn_mask_reformat/input_ids_subgraph/Add_2"Add

�

;/model/attn_mask_reformat/input_ids_subgraph/Add_2/output_0

>/model/attn_mask_reformat/input_ids_subgraph/Concat_3/output_0=/model/attn_mask_reformat/input_ids_subgraph/Reshape/output_04/model/attn_mask_reformat/input_ids_subgraph/Reshape"Reshape

�

;/model/attn_mask_reformat/input_ids_subgraph/Range/output_0

=/model/attn_mask_reformat/input_ids_subgraph/Reshape/output_0:/model/attn_mask_reformat/input_ids_subgraph/Less/output_01/model/attn_mask_reformat/input_ids_subgraph/Less"Less

w/model/constants/FLOAT16/0/model/constant_nodes/FLOAT16/0"Constant*.

value*"

B/model/constants/FLOAT16/0J��

�

:/model/attn_mask_reformat/input_ids_subgraph/Less/output_0

/model/constants/FLOAT16/0

G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_2/output_0=/model/attn_mask_reformat/input_ids_subgraph/Where_2/output_04/model/attn_mask_reformat/input_ids_subgraph/Where_2"Where

�

=/model/attn_mask_reformat/input_ids_subgraph/Where_2/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_8/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_8" Unsqueeze

�

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_8/output_0

/model/constants/INT64/[1]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_9/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_9" Unsqueeze

�

input_ids=/model/attn_mask_reformat/input_ids_subgraph/Shape_1/output_04/model/attn_mask_reformat/input_ids_subgraph/Shape_1"Shape

�

=/model/attn_mask_reformat/input_ids_subgraph/Shape_1/output_0

/model/constants/INT64/0>/model/attn_mask_reformat/input_ids_subgraph/Gather_1/output_05/model/attn_mask_reformat/input_ids_subgraph/Gather_1"Gather*

axis��

�

>/model/attn_mask_reformat/input_ids_subgraph/Gather_1/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_1/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_1" Unsqueeze

�

>/model/attn_mask_reformat/input_ids_subgraph/Gather_2/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_2/output_08/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_2" Unsqueeze

�

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_1/output_0

/model/constants/INT64/[1]

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_2/output_0

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_3/output_0>/model/attn_mask_reformat/input_ids_subgraph/Concat_1/output_05/model/attn_mask_reformat/input_ids_subgraph/Concat_1"Concat*

axis��

�

>/model/attn_mask_reformat/input_ids_subgraph/Concat_1/output_0=/model/attn_mask_reformat/input_ids_subgraph/Shape_3/output_04/model/attn_mask_reformat/input_ids_subgraph/Shape_3"Shape

�

=/model/attn_mask_reformat/input_ids_subgraph/Shape_3/output_0G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_1/output_0>/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_1"ConstantOfShape*

value*J��

z/model/constants/INT64/-1/model/constant_nodes/INT64/-1"Constant*3

value*'B/model/constants/INT64/-1J��

�

G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_1/output_0

/model/constants/INT64/-19/model/attn_mask_reformat/input_ids_subgraph/Mul/output_00/model/attn_mask_reformat/input_ids_subgraph/Mul"Mul

�

>/model/attn_mask_reformat/input_ids_subgraph/Concat_1/output_0

9/model/attn_mask_reformat/input_ids_subgraph/Mul/output_0;/model/attn_mask_reformat/input_ids_subgraph/Equal/output_02/model/attn_mask_reformat/input_ids_subgraph/Equal"Equal

�

;/model/attn_mask_reformat/input_ids_subgraph/Equal/output_0

G/model/attn_mask_reformat/input_ids_subgraph/ConstantOfShape_1/output_0

>/model/attn_mask_reformat/input_ids_subgraph/Concat_1/output_0=/model/attn_mask_reformat/input_ids_subgraph/Where_1/output_04/model/attn_mask_reformat/input_ids_subgraph/Where_1"Where

�

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_9/output_0

=/model/attn_mask_reformat/input_ids_subgraph/Where_1/output_0</model/attn_mask_reformat/input_ids_subgraph/Expand/output_03/model/attn_mask_reformat/input_ids_subgraph/Expand"Expand

�

attention_mask

/model/constants/INT64/[1]A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_3/output_08/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_3" Unsqueeze

/model/constants/INT64/[2]/model/constant_nodes/INT64/[2]"Constant*6

value**B/model/constants/INT64/[2]J��

�

A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_3/output_0

/model/constants/INT64/[2]A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_4/output_08/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_4" Unsqueeze

�

attention_mask=/model/attn_mask_reformat/attn_mask_subgraph/Shape_1/output_04/model/attn_mask_reformat/attn_mask_subgraph/Shape_1"Shape

�

attention_mask=/model/attn_mask_reformat/attn_mask_subgraph/Shape_2/output_04/model/attn_mask_reformat/attn_mask_subgraph/Shape_2"Shape

�

=/model/attn_mask_reformat/attn_mask_subgraph/Shape_1/output_0

/model/constants/INT64/0>/model/attn_mask_reformat/attn_mask_subgraph/Gather_1/output_05/model/attn_mask_reformat/attn_mask_subgraph/Gather_1"Gather*

axis��

�

=/model/attn_mask_reformat/attn_mask_subgraph/Shape_2/output_0

/model/constants/INT64/1>/model/attn_mask_reformat/attn_mask_subgraph/Gather_2/output_05/model/attn_mask_reformat/attn_mask_subgraph/Gather_2"Gather*

axis��

�

>/model/attn_mask_reformat/attn_mask_subgraph/Gather_1/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_1/output_08/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_1" Unsqueeze

�

>/model/attn_mask_reformat/attn_mask_subgraph/Gather_2/output_0

/model/constants/INT64/[0]A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_2/output_08/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_2" Unsqueeze

�

A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_1/output_0

/model/constants/INT64/[1]

A/model/attn_mask_reformat/input_ids_subgraph/Unsqueeze_6/output_0

A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_2/output_0</model/attn_mask_reformat/attn_mask_subgraph/Concat/output_03/model/attn_mask_reformat/attn_mask_subgraph/Concat"Concat*

axis��

�

</model/attn_mask_reformat/attn_mask_subgraph/Concat/output_0=/model/attn_mask_reformat/attn_mask_subgraph/Shape_3/output_04/model/attn_mask_reformat/attn_mask_subgraph/Shape_3"Shape

�

=/model/attn_mask_reformat/attn_mask_subgraph/Shape_3/output_0E/model/attn_mask_reformat/attn_mask_subgraph/ConstantOfShape/output_0</model/attn_mask_reformat/attn_mask_subgraph/ConstantOfShape"ConstantOfShape*

value*J��

�

E/model/attn_mask_reformat/attn_mask_subgraph/ConstantOfShape/output_0

/model/constants/INT64/-19/model/attn_mask_reformat/attn_mask_subgraph/Mul/output_00/model/attn_mask_reformat/attn_mask_subgraph/Mul"Mul

�

</model/attn_mask_reformat/attn_mask_subgraph/Concat/output_0

9/model/attn_mask_reformat/attn_mask_subgraph/Mul/output_0;/model/attn_mask_reformat/attn_mask_subgraph/Equal/output_02/model/attn_mask_reformat/attn_mask_subgraph/Equal"Equal

�

;/model/attn_mask_reformat/attn_mask_subgraph/Equal/output_0

E/model/attn_mask_reformat/attn_mask_subgraph/ConstantOfShape/output_0

</model/attn_mask_reformat/attn_mask_subgraph/Concat/output_0=/model/attn_mask_reformat/attn_mask_subgraph/Where_1/output_04/model/attn_mask_reformat/attn_mask_subgraph/Where_1"Where

�

A/model/attn_mask_reformat/attn_mask_subgraph/Unsqueeze_4/output_0

=/model/attn_mask_reformat/attn_mask_subgraph/Where_1/output_0</model/attn_mask_reformat/attn_mask_subgraph/Expand/output_03/model/attn_mask_reformat/attn_mask_subgraph/Expand"Expand

�

</model/attn_mask_reformat/attn_mask_subgraph/Expand/output_0</model/attn_mask_reformat/attn_mask_subgraph/Cast_1/output_03/model/attn_mask_reformat/attn_mask_subgraph/Cast_1"Cast*

�

w/model/constants/FLOAT16/1/model/constant_nodes/FLOAT16/1"Constant*.

value*"

B/model/constants/FLOAT16/1J�<�

�

/model/constants/FLOAT16/1

</model/attn_mask_reformat/attn_mask_subgraph/Cast_1/output_09/model/attn_mask_reformat/attn_mask_subgraph/Sub/output_00/model/attn_mask_reformat/attn_mask_subgraph/Sub"Sub

�

9/model/attn_mask_reformat/attn_mask_subgraph/Sub/output_0</model/attn_mask_reformat/attn_mask_subgraph/Cast_2/output_03/model/attn_mask_reformat/attn_mask_subgraph/Cast_2"Cast*

to �

�!/model/constants/FLOAT16/-65504.0&/model/constant_nodes/FLOAT16/-65504.0"Constant*5

value*)

B!/model/constants/FLOAT16/-65504.0J��

�

</model/attn_mask_reformat/attn_mask_subgraph/Cast_2/output_0

!/model/constants/FLOAT16/-65504.0

9/model/attn_mask_reformat/attn_mask_subgraph/Sub/output_0=/model/attn_mask_reformat/attn_mask_subgraph/Where_2/output_04/model/attn_mask_reformat/attn_mask_subgraph/Where_2"Where

�

=/model/attn_mask_reformat/attn_mask_subgraph/Where_2/output_0

</model/attn_mask_reformat/input_ids_subgraph/Expand/output_0&/model/attn_mask_reformat/Add/output_0/model/attn_mask_reformat/Add"Add

�$/model/constants/INT64/[1, 32, 1, 1])/model/constant_nodes/INT64/[1, 32, 1, 1]"Constant*X

value*LB$/model/constants/INT64/[1, 32, 1, 1]J ��

�

&/model/attn_mask_reformat/Add/output_0

$/model/constants/INT64/[1, 32, 1, 1]'/model/attn_mask_reformat/Tile/output_0/model/attn_mask_reformat/Tile"Tile

model.embed_tokens.weight

input_ids#/model/embed_tokens/Gather/output_0/model/embed_tokens/Gather"Gather

�

#/model/embed_tokens/Gather/output_0

%model.layers.0.input_layernorm.weight(/model/layers.0/input_layernorm/output_0)/model/layers.0/input_layernorm/LayerNorm"SimplifiedLayerNormalization*

epsilon��'7�*

axis��*

stash_type�

�

(/model/layers.0/input_layernorm/output_0

(model.layers.0.attn.q_proj.MatMul.weight+/model/layers.0/attn/q_proj/MatMul/output_0"/model/layers.0/attn/q_proj/MatMul"MatMul

�

(/model/layers.0/input_layernorm/output_0

(model.layers.0.attn.k_proj.MatMul.weight+/model/layers.0/attn/k_proj/MatMul/output_0"/model/layers.0/attn/k_proj/MatMul"MatMul

�

(/model/layers.0/input_layernorm/output_0

(model.layers.0.attn.v_proj.MatMul.weight+/model/layers.0/attn/v_proj/MatMul/output_0"/model/layers.0/attn/v_proj/MatMul"MatMul

�

+/model/layers.0/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.0/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.0/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.0/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.0/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.0/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.0/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.0/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.0/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.0.key

past_key_values.0.value0/model/layers.0/attn/MultiHeadAttention/output_0 present.0.keypresent.0.value'/model/layers.0/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.0/attn/MultiHeadAttention/output_0

(model.layers.0.attn.o_proj.MatMul.weight+/model/layers.0/attn/o_proj/MatMul/output_0"/model/layers.0/attn/o_proj/MatMul"MatMul

�

#/model/embed_tokens/Gather/output_0

+/model/layers.0/attn/o_proj/MatMul/output_0

.model.layers.0.post_attention_layernorm.weight1/model/layers.0/post_attention_layernorm/output_0��1/model/layers.0/post_attention_layernorm/output_36/model/layers.0/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.0/post_attention_layernorm/output_0

*model.layers.0.mlp.gate_proj.MatMul.weight-/model/layers.0/mlp/gate_proj/MatMul/output_0$/model/layers.0/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.0/post_attention_layernorm/output_0

(model.layers.0.mlp.up_proj.MatMul.weight+/model/layers.0/mlp/up_proj/MatMul/output_0"/model/layers.0/mlp/up_proj/MatMul"MatMul

�

-/model/layers.0/mlp/gate_proj/MatMul/output_0+/model/layers.0/mlp/act_fn/Sigmoid/output_0"/model/layers.0/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.0/mlp/gate_proj/MatMul/output_0

+/model/layers.0/mlp/act_fn/Sigmoid/output_0'/model/layers.0/mlp/act_fn/Mul/output_0/model/layers.0/mlp/act_fn/Mul"Mul

�

'/model/layers.0/mlp/act_fn/Mul/output_0

+/model/layers.0/mlp/up_proj/MatMul/output_0 /model/layers.0/mlp/Mul/output_0/model/layers.0/mlp/Mul"Mul

�

/model/layers.0/mlp/Mul/output_0

*model.layers.0.mlp.down_proj.MatMul.weight-/model/layers.0/mlp/down_proj/MatMul/output_0$/model/layers.0/mlp/down_proj/MatMul"MatMul

�

1/model/layers.0/post_attention_layernorm/output_3

-/model/layers.0/mlp/down_proj/MatMul/output_0

%model.layers.1.input_layernorm.weight(/model/layers.1/input_layernorm/output_0��(/model/layers.1/input_layernorm/output_3-/model/layers.1/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.1/input_layernorm/output_0

(model.layers.1.attn.q_proj.MatMul.weight+/model/layers.1/attn/q_proj/MatMul/output_0"/model/layers.1/attn/q_proj/MatMul"MatMul

�

(/model/layers.1/input_layernorm/output_0

(model.layers.1.attn.k_proj.MatMul.weight+/model/layers.1/attn/k_proj/MatMul/output_0"/model/layers.1/attn/k_proj/MatMul"MatMul

�

(/model/layers.1/input_layernorm/output_0

(model.layers.1.attn.v_proj.MatMul.weight+/model/layers.1/attn/v_proj/MatMul/output_0"/model/layers.1/attn/v_proj/MatMul"MatMul

�

+/model/layers.1/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.1/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.1/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.1/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.1/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.1/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.1/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.1/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.1/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.1.key

past_key_values.1.value0/model/layers.1/attn/MultiHeadAttention/output_0 present.1.keypresent.1.value'/model/layers.1/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.1/attn/MultiHeadAttention/output_0

(model.layers.1.attn.o_proj.MatMul.weight+/model/layers.1/attn/o_proj/MatMul/output_0"/model/layers.1/attn/o_proj/MatMul"MatMul

�

(/model/layers.1/input_layernorm/output_3

+/model/layers.1/attn/o_proj/MatMul/output_0

.model.layers.1.post_attention_layernorm.weight1/model/layers.1/post_attention_layernorm/output_0��1/model/layers.1/post_attention_layernorm/output_36/model/layers.1/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.1/post_attention_layernorm/output_0

*model.layers.1.mlp.gate_proj.MatMul.weight-/model/layers.1/mlp/gate_proj/MatMul/output_0$/model/layers.1/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.1/post_attention_layernorm/output_0

(model.layers.1.mlp.up_proj.MatMul.weight+/model/layers.1/mlp/up_proj/MatMul/output_0"/model/layers.1/mlp/up_proj/MatMul"MatMul

�

-/model/layers.1/mlp/gate_proj/MatMul/output_0+/model/layers.1/mlp/act_fn/Sigmoid/output_0"/model/layers.1/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.1/mlp/gate_proj/MatMul/output_0

+/model/layers.1/mlp/act_fn/Sigmoid/output_0'/model/layers.1/mlp/act_fn/Mul/output_0/model/layers.1/mlp/act_fn/Mul"Mul

�

'/model/layers.1/mlp/act_fn/Mul/output_0

+/model/layers.1/mlp/up_proj/MatMul/output_0 /model/layers.1/mlp/Mul/output_0/model/layers.1/mlp/Mul"Mul

�

/model/layers.1/mlp/Mul/output_0

*model.layers.1.mlp.down_proj.MatMul.weight-/model/layers.1/mlp/down_proj/MatMul/output_0$/model/layers.1/mlp/down_proj/MatMul"MatMul

�

1/model/layers.1/post_attention_layernorm/output_3

-/model/layers.1/mlp/down_proj/MatMul/output_0

%model.layers.2.input_layernorm.weight(/model/layers.2/input_layernorm/output_0��(/model/layers.2/input_layernorm/output_3-/model/layers.2/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.2/input_layernorm/output_0

(model.layers.2.attn.q_proj.MatMul.weight+/model/layers.2/attn/q_proj/MatMul/output_0"/model/layers.2/attn/q_proj/MatMul"MatMul

�

(/model/layers.2/input_layernorm/output_0

(model.layers.2.attn.k_proj.MatMul.weight+/model/layers.2/attn/k_proj/MatMul/output_0"/model/layers.2/attn/k_proj/MatMul"MatMul

�

(/model/layers.2/input_layernorm/output_0

(model.layers.2.attn.v_proj.MatMul.weight+/model/layers.2/attn/v_proj/MatMul/output_0"/model/layers.2/attn/v_proj/MatMul"MatMul

�

+/model/layers.2/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.2/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.2/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.2/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.2/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.2/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.2/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.2/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.2/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.2.key

past_key_values.2.value0/model/layers.2/attn/MultiHeadAttention/output_0 present.2.keypresent.2.value'/model/layers.2/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.2/attn/MultiHeadAttention/output_0

(model.layers.2.attn.o_proj.MatMul.weight+/model/layers.2/attn/o_proj/MatMul/output_0"/model/layers.2/attn/o_proj/MatMul"MatMul

�

(/model/layers.2/input_layernorm/output_3

+/model/layers.2/attn/o_proj/MatMul/output_0

.model.layers.2.post_attention_layernorm.weight1/model/layers.2/post_attention_layernorm/output_0��1/model/layers.2/post_attention_layernorm/output_36/model/layers.2/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.2/post_attention_layernorm/output_0

*model.layers.2.mlp.gate_proj.MatMul.weight-/model/layers.2/mlp/gate_proj/MatMul/output_0$/model/layers.2/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.2/post_attention_layernorm/output_0

(model.layers.2.mlp.up_proj.MatMul.weight+/model/layers.2/mlp/up_proj/MatMul/output_0"/model/layers.2/mlp/up_proj/MatMul"MatMul

�

-/model/layers.2/mlp/gate_proj/MatMul/output_0+/model/layers.2/mlp/act_fn/Sigmoid/output_0"/model/layers.2/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.2/mlp/gate_proj/MatMul/output_0

+/model/layers.2/mlp/act_fn/Sigmoid/output_0'/model/layers.2/mlp/act_fn/Mul/output_0/model/layers.2/mlp/act_fn/Mul"Mul

�

'/model/layers.2/mlp/act_fn/Mul/output_0

+/model/layers.2/mlp/up_proj/MatMul/output_0 /model/layers.2/mlp/Mul/output_0/model/layers.2/mlp/Mul"Mul

�

/model/layers.2/mlp/Mul/output_0

*model.layers.2.mlp.down_proj.MatMul.weight-/model/layers.2/mlp/down_proj/MatMul/output_0$/model/layers.2/mlp/down_proj/MatMul"MatMul

�

1/model/layers.2/post_attention_layernorm/output_3

-/model/layers.2/mlp/down_proj/MatMul/output_0

%model.layers.3.input_layernorm.weight(/model/layers.3/input_layernorm/output_0��(/model/layers.3/input_layernorm/output_3-/model/layers.3/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.3/input_layernorm/output_0

(model.layers.3.attn.q_proj.MatMul.weight+/model/layers.3/attn/q_proj/MatMul/output_0"/model/layers.3/attn/q_proj/MatMul"MatMul

�

(/model/layers.3/input_layernorm/output_0

(model.layers.3.attn.k_proj.MatMul.weight+/model/layers.3/attn/k_proj/MatMul/output_0"/model/layers.3/attn/k_proj/MatMul"MatMul

�

(/model/layers.3/input_layernorm/output_0

(model.layers.3.attn.v_proj.MatMul.weight+/model/layers.3/attn/v_proj/MatMul/output_0"/model/layers.3/attn/v_proj/MatMul"MatMul

�

+/model/layers.3/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.3/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.3/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.3/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.3/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.3/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.3/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.3/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.3/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.3.key

past_key_values.3.value0/model/layers.3/attn/MultiHeadAttention/output_0 present.3.keypresent.3.value'/model/layers.3/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.3/attn/MultiHeadAttention/output_0

(model.layers.3.attn.o_proj.MatMul.weight+/model/layers.3/attn/o_proj/MatMul/output_0"/model/layers.3/attn/o_proj/MatMul"MatMul

�

(/model/layers.3/input_layernorm/output_3

+/model/layers.3/attn/o_proj/MatMul/output_0

.model.layers.3.post_attention_layernorm.weight1/model/layers.3/post_attention_layernorm/output_0��1/model/layers.3/post_attention_layernorm/output_36/model/layers.3/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.3/post_attention_layernorm/output_0

*model.layers.3.mlp.gate_proj.MatMul.weight-/model/layers.3/mlp/gate_proj/MatMul/output_0$/model/layers.3/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.3/post_attention_layernorm/output_0

(model.layers.3.mlp.up_proj.MatMul.weight+/model/layers.3/mlp/up_proj/MatMul/output_0"/model/layers.3/mlp/up_proj/MatMul"MatMul

�

-/model/layers.3/mlp/gate_proj/MatMul/output_0+/model/layers.3/mlp/act_fn/Sigmoid/output_0"/model/layers.3/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.3/mlp/gate_proj/MatMul/output_0

+/model/layers.3/mlp/act_fn/Sigmoid/output_0'/model/layers.3/mlp/act_fn/Mul/output_0/model/layers.3/mlp/act_fn/Mul"Mul

�

'/model/layers.3/mlp/act_fn/Mul/output_0

+/model/layers.3/mlp/up_proj/MatMul/output_0 /model/layers.3/mlp/Mul/output_0/model/layers.3/mlp/Mul"Mul

�

/model/layers.3/mlp/Mul/output_0

*model.layers.3.mlp.down_proj.MatMul.weight-/model/layers.3/mlp/down_proj/MatMul/output_0$/model/layers.3/mlp/down_proj/MatMul"MatMul

�

1/model/layers.3/post_attention_layernorm/output_3

-/model/layers.3/mlp/down_proj/MatMul/output_0

%model.layers.4.input_layernorm.weight(/model/layers.4/input_layernorm/output_0��(/model/layers.4/input_layernorm/output_3-/model/layers.4/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.4/input_layernorm/output_0

(model.layers.4.attn.q_proj.MatMul.weight+/model/layers.4/attn/q_proj/MatMul/output_0"/model/layers.4/attn/q_proj/MatMul"MatMul

�

(/model/layers.4/input_layernorm/output_0

(model.layers.4.attn.k_proj.MatMul.weight+/model/layers.4/attn/k_proj/MatMul/output_0"/model/layers.4/attn/k_proj/MatMul"MatMul

�

(/model/layers.4/input_layernorm/output_0

(model.layers.4.attn.v_proj.MatMul.weight+/model/layers.4/attn/v_proj/MatMul/output_0"/model/layers.4/attn/v_proj/MatMul"MatMul

�

+/model/layers.4/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.4/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.4/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.4/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.4/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.4/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.4/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.4/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.4/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.4.key

past_key_values.4.value0/model/layers.4/attn/MultiHeadAttention/output_0 present.4.keypresent.4.value'/model/layers.4/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.4/attn/MultiHeadAttention/output_0

(model.layers.4.attn.o_proj.MatMul.weight+/model/layers.4/attn/o_proj/MatMul/output_0"/model/layers.4/attn/o_proj/MatMul"MatMul

�

(/model/layers.4/input_layernorm/output_3

+/model/layers.4/attn/o_proj/MatMul/output_0

.model.layers.4.post_attention_layernorm.weight1/model/layers.4/post_attention_layernorm/output_0��1/model/layers.4/post_attention_layernorm/output_36/model/layers.4/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.4/post_attention_layernorm/output_0

*model.layers.4.mlp.gate_proj.MatMul.weight-/model/layers.4/mlp/gate_proj/MatMul/output_0$/model/layers.4/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.4/post_attention_layernorm/output_0

(model.layers.4.mlp.up_proj.MatMul.weight+/model/layers.4/mlp/up_proj/MatMul/output_0"/model/layers.4/mlp/up_proj/MatMul"MatMul

�

-/model/layers.4/mlp/gate_proj/MatMul/output_0+/model/layers.4/mlp/act_fn/Sigmoid/output_0"/model/layers.4/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.4/mlp/gate_proj/MatMul/output_0

+/model/layers.4/mlp/act_fn/Sigmoid/output_0'/model/layers.4/mlp/act_fn/Mul/output_0/model/layers.4/mlp/act_fn/Mul"Mul

�

'/model/layers.4/mlp/act_fn/Mul/output_0

+/model/layers.4/mlp/up_proj/MatMul/output_0 /model/layers.4/mlp/Mul/output_0/model/layers.4/mlp/Mul"Mul

�

/model/layers.4/mlp/Mul/output_0

*model.layers.4.mlp.down_proj.MatMul.weight-/model/layers.4/mlp/down_proj/MatMul/output_0$/model/layers.4/mlp/down_proj/MatMul"MatMul

�

1/model/layers.4/post_attention_layernorm/output_3

-/model/layers.4/mlp/down_proj/MatMul/output_0

%model.layers.5.input_layernorm.weight(/model/layers.5/input_layernorm/output_0��(/model/layers.5/input_layernorm/output_3-/model/layers.5/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.5/input_layernorm/output_0

(model.layers.5.attn.q_proj.MatMul.weight+/model/layers.5/attn/q_proj/MatMul/output_0"/model/layers.5/attn/q_proj/MatMul"MatMul

�

(/model/layers.5/input_layernorm/output_0

(model.layers.5.attn.k_proj.MatMul.weight+/model/layers.5/attn/k_proj/MatMul/output_0"/model/layers.5/attn/k_proj/MatMul"MatMul

�

(/model/layers.5/input_layernorm/output_0

(model.layers.5.attn.v_proj.MatMul.weight+/model/layers.5/attn/v_proj/MatMul/output_0"/model/layers.5/attn/v_proj/MatMul"MatMul

�

+/model/layers.5/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.5/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.5/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.5/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.5/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.5/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.5/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.5/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.5/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.5.key

past_key_values.5.value0/model/layers.5/attn/MultiHeadAttention/output_0 present.5.keypresent.5.value'/model/layers.5/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.5/attn/MultiHeadAttention/output_0

(model.layers.5.attn.o_proj.MatMul.weight+/model/layers.5/attn/o_proj/MatMul/output_0"/model/layers.5/attn/o_proj/MatMul"MatMul

�

(/model/layers.5/input_layernorm/output_3

+/model/layers.5/attn/o_proj/MatMul/output_0

.model.layers.5.post_attention_layernorm.weight1/model/layers.5/post_attention_layernorm/output_0��1/model/layers.5/post_attention_layernorm/output_36/model/layers.5/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.5/post_attention_layernorm/output_0

*model.layers.5.mlp.gate_proj.MatMul.weight-/model/layers.5/mlp/gate_proj/MatMul/output_0$/model/layers.5/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.5/post_attention_layernorm/output_0

(model.layers.5.mlp.up_proj.MatMul.weight+/model/layers.5/mlp/up_proj/MatMul/output_0"/model/layers.5/mlp/up_proj/MatMul"MatMul

�

-/model/layers.5/mlp/gate_proj/MatMul/output_0+/model/layers.5/mlp/act_fn/Sigmoid/output_0"/model/layers.5/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.5/mlp/gate_proj/MatMul/output_0

+/model/layers.5/mlp/act_fn/Sigmoid/output_0'/model/layers.5/mlp/act_fn/Mul/output_0/model/layers.5/mlp/act_fn/Mul"Mul

�

'/model/layers.5/mlp/act_fn/Mul/output_0

+/model/layers.5/mlp/up_proj/MatMul/output_0 /model/layers.5/mlp/Mul/output_0/model/layers.5/mlp/Mul"Mul

�

/model/layers.5/mlp/Mul/output_0

*model.layers.5.mlp.down_proj.MatMul.weight-/model/layers.5/mlp/down_proj/MatMul/output_0$/model/layers.5/mlp/down_proj/MatMul"MatMul

�

1/model/layers.5/post_attention_layernorm/output_3

-/model/layers.5/mlp/down_proj/MatMul/output_0

%model.layers.6.input_layernorm.weight(/model/layers.6/input_layernorm/output_0��(/model/layers.6/input_layernorm/output_3-/model/layers.6/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.6/input_layernorm/output_0

(model.layers.6.attn.q_proj.MatMul.weight+/model/layers.6/attn/q_proj/MatMul/output_0"/model/layers.6/attn/q_proj/MatMul"MatMul

�

(/model/layers.6/input_layernorm/output_0

(model.layers.6.attn.k_proj.MatMul.weight+/model/layers.6/attn/k_proj/MatMul/output_0"/model/layers.6/attn/k_proj/MatMul"MatMul

�

(/model/layers.6/input_layernorm/output_0

(model.layers.6.attn.v_proj.MatMul.weight+/model/layers.6/attn/v_proj/MatMul/output_0"/model/layers.6/attn/v_proj/MatMul"MatMul

�

+/model/layers.6/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.6/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.6/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.6/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.6/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.6/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.6/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.6/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.6/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.6.key

past_key_values.6.value0/model/layers.6/attn/MultiHeadAttention/output_0 present.6.keypresent.6.value'/model/layers.6/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.6/attn/MultiHeadAttention/output_0

(model.layers.6.attn.o_proj.MatMul.weight+/model/layers.6/attn/o_proj/MatMul/output_0"/model/layers.6/attn/o_proj/MatMul"MatMul

�

(/model/layers.6/input_layernorm/output_3

+/model/layers.6/attn/o_proj/MatMul/output_0

.model.layers.6.post_attention_layernorm.weight1/model/layers.6/post_attention_layernorm/output_0��1/model/layers.6/post_attention_layernorm/output_36/model/layers.6/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.6/post_attention_layernorm/output_0

*model.layers.6.mlp.gate_proj.MatMul.weight-/model/layers.6/mlp/gate_proj/MatMul/output_0$/model/layers.6/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.6/post_attention_layernorm/output_0

(model.layers.6.mlp.up_proj.MatMul.weight+/model/layers.6/mlp/up_proj/MatMul/output_0"/model/layers.6/mlp/up_proj/MatMul"MatMul

�

-/model/layers.6/mlp/gate_proj/MatMul/output_0+/model/layers.6/mlp/act_fn/Sigmoid/output_0"/model/layers.6/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.6/mlp/gate_proj/MatMul/output_0

+/model/layers.6/mlp/act_fn/Sigmoid/output_0'/model/layers.6/mlp/act_fn/Mul/output_0/model/layers.6/mlp/act_fn/Mul"Mul

�

'/model/layers.6/mlp/act_fn/Mul/output_0

+/model/layers.6/mlp/up_proj/MatMul/output_0 /model/layers.6/mlp/Mul/output_0/model/layers.6/mlp/Mul"Mul

�

/model/layers.6/mlp/Mul/output_0

*model.layers.6.mlp.down_proj.MatMul.weight-/model/layers.6/mlp/down_proj/MatMul/output_0$/model/layers.6/mlp/down_proj/MatMul"MatMul

�

1/model/layers.6/post_attention_layernorm/output_3

-/model/layers.6/mlp/down_proj/MatMul/output_0

%model.layers.7.input_layernorm.weight(/model/layers.7/input_layernorm/output_0��(/model/layers.7/input_layernorm/output_3-/model/layers.7/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.7/input_layernorm/output_0

(model.layers.7.attn.q_proj.MatMul.weight+/model/layers.7/attn/q_proj/MatMul/output_0"/model/layers.7/attn/q_proj/MatMul"MatMul

�

(/model/layers.7/input_layernorm/output_0

(model.layers.7.attn.k_proj.MatMul.weight+/model/layers.7/attn/k_proj/MatMul/output_0"/model/layers.7/attn/k_proj/MatMul"MatMul

�

(/model/layers.7/input_layernorm/output_0

(model.layers.7.attn.v_proj.MatMul.weight+/model/layers.7/attn/v_proj/MatMul/output_0"/model/layers.7/attn/v_proj/MatMul"MatMul

�

+/model/layers.7/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.7/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.7/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.7/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.7/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.7/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.7/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.7/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.7/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.7.key

past_key_values.7.value0/model/layers.7/attn/MultiHeadAttention/output_0 present.7.keypresent.7.value'/model/layers.7/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.7/attn/MultiHeadAttention/output_0

(model.layers.7.attn.o_proj.MatMul.weight+/model/layers.7/attn/o_proj/MatMul/output_0"/model/layers.7/attn/o_proj/MatMul"MatMul

�

(/model/layers.7/input_layernorm/output_3

+/model/layers.7/attn/o_proj/MatMul/output_0

.model.layers.7.post_attention_layernorm.weight1/model/layers.7/post_attention_layernorm/output_0��1/model/layers.7/post_attention_layernorm/output_36/model/layers.7/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.7/post_attention_layernorm/output_0

*model.layers.7.mlp.gate_proj.MatMul.weight-/model/layers.7/mlp/gate_proj/MatMul/output_0$/model/layers.7/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.7/post_attention_layernorm/output_0

(model.layers.7.mlp.up_proj.MatMul.weight+/model/layers.7/mlp/up_proj/MatMul/output_0"/model/layers.7/mlp/up_proj/MatMul"MatMul

�

-/model/layers.7/mlp/gate_proj/MatMul/output_0+/model/layers.7/mlp/act_fn/Sigmoid/output_0"/model/layers.7/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.7/mlp/gate_proj/MatMul/output_0

+/model/layers.7/mlp/act_fn/Sigmoid/output_0'/model/layers.7/mlp/act_fn/Mul/output_0/model/layers.7/mlp/act_fn/Mul"Mul

�

'/model/layers.7/mlp/act_fn/Mul/output_0

+/model/layers.7/mlp/up_proj/MatMul/output_0 /model/layers.7/mlp/Mul/output_0/model/layers.7/mlp/Mul"Mul

�

/model/layers.7/mlp/Mul/output_0

*model.layers.7.mlp.down_proj.MatMul.weight-/model/layers.7/mlp/down_proj/MatMul/output_0$/model/layers.7/mlp/down_proj/MatMul"MatMul

�

1/model/layers.7/post_attention_layernorm/output_3

-/model/layers.7/mlp/down_proj/MatMul/output_0

%model.layers.8.input_layernorm.weight(/model/layers.8/input_layernorm/output_0��(/model/layers.8/input_layernorm/output_3-/model/layers.8/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.8/input_layernorm/output_0

(model.layers.8.attn.q_proj.MatMul.weight+/model/layers.8/attn/q_proj/MatMul/output_0"/model/layers.8/attn/q_proj/MatMul"MatMul

�

(/model/layers.8/input_layernorm/output_0

(model.layers.8.attn.k_proj.MatMul.weight+/model/layers.8/attn/k_proj/MatMul/output_0"/model/layers.8/attn/k_proj/MatMul"MatMul

�

(/model/layers.8/input_layernorm/output_0

(model.layers.8.attn.v_proj.MatMul.weight+/model/layers.8/attn/v_proj/MatMul/output_0"/model/layers.8/attn/v_proj/MatMul"MatMul

�

+/model/layers.8/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.8/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.8/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.8/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.8/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.8/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.8/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.8/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.8/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.8.key

past_key_values.8.value0/model/layers.8/attn/MultiHeadAttention/output_0 present.8.keypresent.8.value'/model/layers.8/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.8/attn/MultiHeadAttention/output_0

(model.layers.8.attn.o_proj.MatMul.weight+/model/layers.8/attn/o_proj/MatMul/output_0"/model/layers.8/attn/o_proj/MatMul"MatMul

�

(/model/layers.8/input_layernorm/output_3

+/model/layers.8/attn/o_proj/MatMul/output_0

.model.layers.8.post_attention_layernorm.weight1/model/layers.8/post_attention_layernorm/output_0��1/model/layers.8/post_attention_layernorm/output_36/model/layers.8/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.8/post_attention_layernorm/output_0

*model.layers.8.mlp.gate_proj.MatMul.weight-/model/layers.8/mlp/gate_proj/MatMul/output_0$/model/layers.8/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.8/post_attention_layernorm/output_0

(model.layers.8.mlp.up_proj.MatMul.weight+/model/layers.8/mlp/up_proj/MatMul/output_0"/model/layers.8/mlp/up_proj/MatMul"MatMul

�

-/model/layers.8/mlp/gate_proj/MatMul/output_0+/model/layers.8/mlp/act_fn/Sigmoid/output_0"/model/layers.8/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.8/mlp/gate_proj/MatMul/output_0

+/model/layers.8/mlp/act_fn/Sigmoid/output_0'/model/layers.8/mlp/act_fn/Mul/output_0/model/layers.8/mlp/act_fn/Mul"Mul

�

'/model/layers.8/mlp/act_fn/Mul/output_0

+/model/layers.8/mlp/up_proj/MatMul/output_0 /model/layers.8/mlp/Mul/output_0/model/layers.8/mlp/Mul"Mul

�

/model/layers.8/mlp/Mul/output_0

*model.layers.8.mlp.down_proj.MatMul.weight-/model/layers.8/mlp/down_proj/MatMul/output_0$/model/layers.8/mlp/down_proj/MatMul"MatMul

�

1/model/layers.8/post_attention_layernorm/output_3

-/model/layers.8/mlp/down_proj/MatMul/output_0

%model.layers.9.input_layernorm.weight(/model/layers.9/input_layernorm/output_0��(/model/layers.9/input_layernorm/output_3-/model/layers.9/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

(/model/layers.9/input_layernorm/output_0

(model.layers.9.attn.q_proj.MatMul.weight+/model/layers.9/attn/q_proj/MatMul/output_0"/model/layers.9/attn/q_proj/MatMul"MatMul

�

(/model/layers.9/input_layernorm/output_0

(model.layers.9.attn.k_proj.MatMul.weight+/model/layers.9/attn/k_proj/MatMul/output_0"/model/layers.9/attn/k_proj/MatMul"MatMul

�

(/model/layers.9/input_layernorm/output_0

(model.layers.9.attn.v_proj.MatMul.weight+/model/layers.9/attn/v_proj/MatMul/output_0"/model/layers.9/attn/v_proj/MatMul"MatMul

�

+/model/layers.9/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.9/attn/q_rotary/RotaryEmbedding/output_0-/model/layers.9/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

+/model/layers.9/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache6/model/layers.9/attn/k_rotary/RotaryEmbedding/output_0-/model/layers.9/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

6/model/layers.9/attn/q_rotary/RotaryEmbedding/output_0

6/model/layers.9/attn/k_rotary/RotaryEmbedding/output_0

+/model/layers.9/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.9.key

past_key_values.9.value0/model/layers.9/attn/MultiHeadAttention/output_0 present.9.keypresent.9.value'/model/layers.9/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

0/model/layers.9/attn/MultiHeadAttention/output_0

(model.layers.9.attn.o_proj.MatMul.weight+/model/layers.9/attn/o_proj/MatMul/output_0"/model/layers.9/attn/o_proj/MatMul"MatMul

�

(/model/layers.9/input_layernorm/output_3

+/model/layers.9/attn/o_proj/MatMul/output_0

.model.layers.9.post_attention_layernorm.weight1/model/layers.9/post_attention_layernorm/output_0��1/model/layers.9/post_attention_layernorm/output_36/model/layers.9/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

1/model/layers.9/post_attention_layernorm/output_0

*model.layers.9.mlp.gate_proj.MatMul.weight-/model/layers.9/mlp/gate_proj/MatMul/output_0$/model/layers.9/mlp/gate_proj/MatMul"MatMul

�

1/model/layers.9/post_attention_layernorm/output_0

(model.layers.9.mlp.up_proj.MatMul.weight+/model/layers.9/mlp/up_proj/MatMul/output_0"/model/layers.9/mlp/up_proj/MatMul"MatMul

�

-/model/layers.9/mlp/gate_proj/MatMul/output_0+/model/layers.9/mlp/act_fn/Sigmoid/output_0"/model/layers.9/mlp/act_fn/Sigmoid"Sigmoid

�

-/model/layers.9/mlp/gate_proj/MatMul/output_0

+/model/layers.9/mlp/act_fn/Sigmoid/output_0'/model/layers.9/mlp/act_fn/Mul/output_0/model/layers.9/mlp/act_fn/Mul"Mul

�

'/model/layers.9/mlp/act_fn/Mul/output_0

+/model/layers.9/mlp/up_proj/MatMul/output_0 /model/layers.9/mlp/Mul/output_0/model/layers.9/mlp/Mul"Mul

�

/model/layers.9/mlp/Mul/output_0

*model.layers.9.mlp.down_proj.MatMul.weight-/model/layers.9/mlp/down_proj/MatMul/output_0$/model/layers.9/mlp/down_proj/MatMul"MatMul

�

1/model/layers.9/post_attention_layernorm/output_3

-/model/layers.9/mlp/down_proj/MatMul/output_0

&model.layers.10.input_layernorm.weight)/model/layers.10/input_layernorm/output_0��)/model/layers.10/input_layernorm/output_3./model/layers.10/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.10/input_layernorm/output_0

)model.layers.10.attn.q_proj.MatMul.weight,/model/layers.10/attn/q_proj/MatMul/output_0#/model/layers.10/attn/q_proj/MatMul"MatMul

�

)/model/layers.10/input_layernorm/output_0

)model.layers.10.attn.k_proj.MatMul.weight,/model/layers.10/attn/k_proj/MatMul/output_0#/model/layers.10/attn/k_proj/MatMul"MatMul

�

)/model/layers.10/input_layernorm/output_0

)model.layers.10.attn.v_proj.MatMul.weight,/model/layers.10/attn/v_proj/MatMul/output_0#/model/layers.10/attn/v_proj/MatMul"MatMul

�

,/model/layers.10/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.10/attn/q_rotary/RotaryEmbedding/output_0./model/layers.10/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.10/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.10/attn/k_rotary/RotaryEmbedding/output_0./model/layers.10/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.10/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.10/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.10/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.10.key

past_key_values.10.value1/model/layers.10/attn/MultiHeadAttention/output_0present.10.keypresent.10.value(/model/layers.10/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.10/attn/MultiHeadAttention/output_0

)model.layers.10.attn.o_proj.MatMul.weight,/model/layers.10/attn/o_proj/MatMul/output_0#/model/layers.10/attn/o_proj/MatMul"MatMul

�

)/model/layers.10/input_layernorm/output_3

,/model/layers.10/attn/o_proj/MatMul/output_0

/model.layers.10.post_attention_layernorm.weight2/model/layers.10/post_attention_layernorm/output_0��2/model/layers.10/post_attention_layernorm/output_37/model/layers.10/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.10/post_attention_layernorm/output_0

+model.layers.10.mlp.gate_proj.MatMul.weight./model/layers.10/mlp/gate_proj/MatMul/output_0%/model/layers.10/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.10/post_attention_layernorm/output_0

)model.layers.10.mlp.up_proj.MatMul.weight,/model/layers.10/mlp/up_proj/MatMul/output_0#/model/layers.10/mlp/up_proj/MatMul"MatMul

�

./model/layers.10/mlp/gate_proj/MatMul/output_0,/model/layers.10/mlp/act_fn/Sigmoid/output_0#/model/layers.10/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.10/mlp/gate_proj/MatMul/output_0

,/model/layers.10/mlp/act_fn/Sigmoid/output_0(/model/layers.10/mlp/act_fn/Mul/output_0/model/layers.10/mlp/act_fn/Mul"Mul

�

(/model/layers.10/mlp/act_fn/Mul/output_0

,/model/layers.10/mlp/up_proj/MatMul/output_0!/model/layers.10/mlp/Mul/output_0/model/layers.10/mlp/Mul"Mul

�

!/model/layers.10/mlp/Mul/output_0

+model.layers.10.mlp.down_proj.MatMul.weight./model/layers.10/mlp/down_proj/MatMul/output_0%/model/layers.10/mlp/down_proj/MatMul"MatMul

�

2/model/layers.10/post_attention_layernorm/output_3

./model/layers.10/mlp/down_proj/MatMul/output_0

&model.layers.11.input_layernorm.weight)/model/layers.11/input_layernorm/output_0��)/model/layers.11/input_layernorm/output_3./model/layers.11/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.11/input_layernorm/output_0

)model.layers.11.attn.q_proj.MatMul.weight,/model/layers.11/attn/q_proj/MatMul/output_0#/model/layers.11/attn/q_proj/MatMul"MatMul

�

)/model/layers.11/input_layernorm/output_0

)model.layers.11.attn.k_proj.MatMul.weight,/model/layers.11/attn/k_proj/MatMul/output_0#/model/layers.11/attn/k_proj/MatMul"MatMul

�

)/model/layers.11/input_layernorm/output_0

)model.layers.11.attn.v_proj.MatMul.weight,/model/layers.11/attn/v_proj/MatMul/output_0#/model/layers.11/attn/v_proj/MatMul"MatMul

�

,/model/layers.11/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.11/attn/q_rotary/RotaryEmbedding/output_0./model/layers.11/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.11/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.11/attn/k_rotary/RotaryEmbedding/output_0./model/layers.11/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.11/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.11/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.11/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.11.key

past_key_values.11.value1/model/layers.11/attn/MultiHeadAttention/output_0present.11.keypresent.11.value(/model/layers.11/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.11/attn/MultiHeadAttention/output_0

)model.layers.11.attn.o_proj.MatMul.weight,/model/layers.11/attn/o_proj/MatMul/output_0#/model/layers.11/attn/o_proj/MatMul"MatMul

�

)/model/layers.11/input_layernorm/output_3

,/model/layers.11/attn/o_proj/MatMul/output_0

/model.layers.11.post_attention_layernorm.weight2/model/layers.11/post_attention_layernorm/output_0��2/model/layers.11/post_attention_layernorm/output_37/model/layers.11/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.11/post_attention_layernorm/output_0

+model.layers.11.mlp.gate_proj.MatMul.weight./model/layers.11/mlp/gate_proj/MatMul/output_0%/model/layers.11/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.11/post_attention_layernorm/output_0

)model.layers.11.mlp.up_proj.MatMul.weight,/model/layers.11/mlp/up_proj/MatMul/output_0#/model/layers.11/mlp/up_proj/MatMul"MatMul

�

./model/layers.11/mlp/gate_proj/MatMul/output_0,/model/layers.11/mlp/act_fn/Sigmoid/output_0#/model/layers.11/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.11/mlp/gate_proj/MatMul/output_0

,/model/layers.11/mlp/act_fn/Sigmoid/output_0(/model/layers.11/mlp/act_fn/Mul/output_0/model/layers.11/mlp/act_fn/Mul"Mul

�

(/model/layers.11/mlp/act_fn/Mul/output_0

,/model/layers.11/mlp/up_proj/MatMul/output_0!/model/layers.11/mlp/Mul/output_0/model/layers.11/mlp/Mul"Mul

�

!/model/layers.11/mlp/Mul/output_0

+model.layers.11.mlp.down_proj.MatMul.weight./model/layers.11/mlp/down_proj/MatMul/output_0%/model/layers.11/mlp/down_proj/MatMul"MatMul

�

2/model/layers.11/post_attention_layernorm/output_3

./model/layers.11/mlp/down_proj/MatMul/output_0

&model.layers.12.input_layernorm.weight)/model/layers.12/input_layernorm/output_0��)/model/layers.12/input_layernorm/output_3./model/layers.12/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.12/input_layernorm/output_0

)model.layers.12.attn.q_proj.MatMul.weight,/model/layers.12/attn/q_proj/MatMul/output_0#/model/layers.12/attn/q_proj/MatMul"MatMul

�

)/model/layers.12/input_layernorm/output_0

)model.layers.12.attn.k_proj.MatMul.weight,/model/layers.12/attn/k_proj/MatMul/output_0#/model/layers.12/attn/k_proj/MatMul"MatMul

�

)/model/layers.12/input_layernorm/output_0

)model.layers.12.attn.v_proj.MatMul.weight,/model/layers.12/attn/v_proj/MatMul/output_0#/model/layers.12/attn/v_proj/MatMul"MatMul

�

,/model/layers.12/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.12/attn/q_rotary/RotaryEmbedding/output_0./model/layers.12/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.12/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.12/attn/k_rotary/RotaryEmbedding/output_0./model/layers.12/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.12/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.12/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.12/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.12.key

past_key_values.12.value1/model/layers.12/attn/MultiHeadAttention/output_0present.12.keypresent.12.value(/model/layers.12/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.12/attn/MultiHeadAttention/output_0

)model.layers.12.attn.o_proj.MatMul.weight,/model/layers.12/attn/o_proj/MatMul/output_0#/model/layers.12/attn/o_proj/MatMul"MatMul

�

)/model/layers.12/input_layernorm/output_3

,/model/layers.12/attn/o_proj/MatMul/output_0

/model.layers.12.post_attention_layernorm.weight2/model/layers.12/post_attention_layernorm/output_0��2/model/layers.12/post_attention_layernorm/output_37/model/layers.12/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.12/post_attention_layernorm/output_0

+model.layers.12.mlp.gate_proj.MatMul.weight./model/layers.12/mlp/gate_proj/MatMul/output_0%/model/layers.12/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.12/post_attention_layernorm/output_0

)model.layers.12.mlp.up_proj.MatMul.weight,/model/layers.12/mlp/up_proj/MatMul/output_0#/model/layers.12/mlp/up_proj/MatMul"MatMul

�

./model/layers.12/mlp/gate_proj/MatMul/output_0,/model/layers.12/mlp/act_fn/Sigmoid/output_0#/model/layers.12/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.12/mlp/gate_proj/MatMul/output_0

,/model/layers.12/mlp/act_fn/Sigmoid/output_0(/model/layers.12/mlp/act_fn/Mul/output_0/model/layers.12/mlp/act_fn/Mul"Mul

�

(/model/layers.12/mlp/act_fn/Mul/output_0

,/model/layers.12/mlp/up_proj/MatMul/output_0!/model/layers.12/mlp/Mul/output_0/model/layers.12/mlp/Mul"Mul

�

!/model/layers.12/mlp/Mul/output_0

+model.layers.12.mlp.down_proj.MatMul.weight./model/layers.12/mlp/down_proj/MatMul/output_0%/model/layers.12/mlp/down_proj/MatMul"MatMul

�

2/model/layers.12/post_attention_layernorm/output_3

./model/layers.12/mlp/down_proj/MatMul/output_0

&model.layers.13.input_layernorm.weight)/model/layers.13/input_layernorm/output_0��)/model/layers.13/input_layernorm/output_3./model/layers.13/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.13/input_layernorm/output_0

)model.layers.13.attn.q_proj.MatMul.weight,/model/layers.13/attn/q_proj/MatMul/output_0#/model/layers.13/attn/q_proj/MatMul"MatMul

�

)/model/layers.13/input_layernorm/output_0

)model.layers.13.attn.k_proj.MatMul.weight,/model/layers.13/attn/k_proj/MatMul/output_0#/model/layers.13/attn/k_proj/MatMul"MatMul

�

)/model/layers.13/input_layernorm/output_0

)model.layers.13.attn.v_proj.MatMul.weight,/model/layers.13/attn/v_proj/MatMul/output_0#/model/layers.13/attn/v_proj/MatMul"MatMul

�

,/model/layers.13/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.13/attn/q_rotary/RotaryEmbedding/output_0./model/layers.13/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.13/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.13/attn/k_rotary/RotaryEmbedding/output_0./model/layers.13/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.13/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.13/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.13/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.13.key

past_key_values.13.value1/model/layers.13/attn/MultiHeadAttention/output_0present.13.keypresent.13.value(/model/layers.13/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.13/attn/MultiHeadAttention/output_0

)model.layers.13.attn.o_proj.MatMul.weight,/model/layers.13/attn/o_proj/MatMul/output_0#/model/layers.13/attn/o_proj/MatMul"MatMul

�

)/model/layers.13/input_layernorm/output_3

,/model/layers.13/attn/o_proj/MatMul/output_0

/model.layers.13.post_attention_layernorm.weight2/model/layers.13/post_attention_layernorm/output_0��2/model/layers.13/post_attention_layernorm/output_37/model/layers.13/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.13/post_attention_layernorm/output_0

+model.layers.13.mlp.gate_proj.MatMul.weight./model/layers.13/mlp/gate_proj/MatMul/output_0%/model/layers.13/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.13/post_attention_layernorm/output_0

)model.layers.13.mlp.up_proj.MatMul.weight,/model/layers.13/mlp/up_proj/MatMul/output_0#/model/layers.13/mlp/up_proj/MatMul"MatMul

�

./model/layers.13/mlp/gate_proj/MatMul/output_0,/model/layers.13/mlp/act_fn/Sigmoid/output_0#/model/layers.13/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.13/mlp/gate_proj/MatMul/output_0

,/model/layers.13/mlp/act_fn/Sigmoid/output_0(/model/layers.13/mlp/act_fn/Mul/output_0/model/layers.13/mlp/act_fn/Mul"Mul

�

(/model/layers.13/mlp/act_fn/Mul/output_0

,/model/layers.13/mlp/up_proj/MatMul/output_0!/model/layers.13/mlp/Mul/output_0/model/layers.13/mlp/Mul"Mul

�

!/model/layers.13/mlp/Mul/output_0

+model.layers.13.mlp.down_proj.MatMul.weight./model/layers.13/mlp/down_proj/MatMul/output_0%/model/layers.13/mlp/down_proj/MatMul"MatMul

�

2/model/layers.13/post_attention_layernorm/output_3

./model/layers.13/mlp/down_proj/MatMul/output_0

&model.layers.14.input_layernorm.weight)/model/layers.14/input_layernorm/output_0��)/model/layers.14/input_layernorm/output_3./model/layers.14/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.14/input_layernorm/output_0

)model.layers.14.attn.q_proj.MatMul.weight,/model/layers.14/attn/q_proj/MatMul/output_0#/model/layers.14/attn/q_proj/MatMul"MatMul

�

)/model/layers.14/input_layernorm/output_0

)model.layers.14.attn.k_proj.MatMul.weight,/model/layers.14/attn/k_proj/MatMul/output_0#/model/layers.14/attn/k_proj/MatMul"MatMul

�

)/model/layers.14/input_layernorm/output_0

)model.layers.14.attn.v_proj.MatMul.weight,/model/layers.14/attn/v_proj/MatMul/output_0#/model/layers.14/attn/v_proj/MatMul"MatMul

�

,/model/layers.14/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.14/attn/q_rotary/RotaryEmbedding/output_0./model/layers.14/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.14/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.14/attn/k_rotary/RotaryEmbedding/output_0./model/layers.14/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.14/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.14/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.14/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.14.key

past_key_values.14.value1/model/layers.14/attn/MultiHeadAttention/output_0present.14.keypresent.14.value(/model/layers.14/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.14/attn/MultiHeadAttention/output_0

)model.layers.14.attn.o_proj.MatMul.weight,/model/layers.14/attn/o_proj/MatMul/output_0#/model/layers.14/attn/o_proj/MatMul"MatMul

�

)/model/layers.14/input_layernorm/output_3

,/model/layers.14/attn/o_proj/MatMul/output_0

/model.layers.14.post_attention_layernorm.weight2/model/layers.14/post_attention_layernorm/output_0��2/model/layers.14/post_attention_layernorm/output_37/model/layers.14/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.14/post_attention_layernorm/output_0

+model.layers.14.mlp.gate_proj.MatMul.weight./model/layers.14/mlp/gate_proj/MatMul/output_0%/model/layers.14/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.14/post_attention_layernorm/output_0

)model.layers.14.mlp.up_proj.MatMul.weight,/model/layers.14/mlp/up_proj/MatMul/output_0#/model/layers.14/mlp/up_proj/MatMul"MatMul

�

./model/layers.14/mlp/gate_proj/MatMul/output_0,/model/layers.14/mlp/act_fn/Sigmoid/output_0#/model/layers.14/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.14/mlp/gate_proj/MatMul/output_0

,/model/layers.14/mlp/act_fn/Sigmoid/output_0(/model/layers.14/mlp/act_fn/Mul/output_0/model/layers.14/mlp/act_fn/Mul"Mul

�

(/model/layers.14/mlp/act_fn/Mul/output_0

,/model/layers.14/mlp/up_proj/MatMul/output_0!/model/layers.14/mlp/Mul/output_0/model/layers.14/mlp/Mul"Mul

�

!/model/layers.14/mlp/Mul/output_0

+model.layers.14.mlp.down_proj.MatMul.weight./model/layers.14/mlp/down_proj/MatMul/output_0%/model/layers.14/mlp/down_proj/MatMul"MatMul

�

2/model/layers.14/post_attention_layernorm/output_3

./model/layers.14/mlp/down_proj/MatMul/output_0

&model.layers.15.input_layernorm.weight)/model/layers.15/input_layernorm/output_0��)/model/layers.15/input_layernorm/output_3./model/layers.15/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.15/input_layernorm/output_0

)model.layers.15.attn.q_proj.MatMul.weight,/model/layers.15/attn/q_proj/MatMul/output_0#/model/layers.15/attn/q_proj/MatMul"MatMul

�

)/model/layers.15/input_layernorm/output_0

)model.layers.15.attn.k_proj.MatMul.weight,/model/layers.15/attn/k_proj/MatMul/output_0#/model/layers.15/attn/k_proj/MatMul"MatMul

�

)/model/layers.15/input_layernorm/output_0

)model.layers.15.attn.v_proj.MatMul.weight,/model/layers.15/attn/v_proj/MatMul/output_0#/model/layers.15/attn/v_proj/MatMul"MatMul

�

,/model/layers.15/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.15/attn/q_rotary/RotaryEmbedding/output_0./model/layers.15/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.15/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.15/attn/k_rotary/RotaryEmbedding/output_0./model/layers.15/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.15/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.15/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.15/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.15.key

past_key_values.15.value1/model/layers.15/attn/MultiHeadAttention/output_0present.15.keypresent.15.value(/model/layers.15/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.15/attn/MultiHeadAttention/output_0

)model.layers.15.attn.o_proj.MatMul.weight,/model/layers.15/attn/o_proj/MatMul/output_0#/model/layers.15/attn/o_proj/MatMul"MatMul

�

)/model/layers.15/input_layernorm/output_3

,/model/layers.15/attn/o_proj/MatMul/output_0

/model.layers.15.post_attention_layernorm.weight2/model/layers.15/post_attention_layernorm/output_0��2/model/layers.15/post_attention_layernorm/output_37/model/layers.15/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.15/post_attention_layernorm/output_0

+model.layers.15.mlp.gate_proj.MatMul.weight./model/layers.15/mlp/gate_proj/MatMul/output_0%/model/layers.15/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.15/post_attention_layernorm/output_0

)model.layers.15.mlp.up_proj.MatMul.weight,/model/layers.15/mlp/up_proj/MatMul/output_0#/model/layers.15/mlp/up_proj/MatMul"MatMul

�

./model/layers.15/mlp/gate_proj/MatMul/output_0,/model/layers.15/mlp/act_fn/Sigmoid/output_0#/model/layers.15/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.15/mlp/gate_proj/MatMul/output_0

,/model/layers.15/mlp/act_fn/Sigmoid/output_0(/model/layers.15/mlp/act_fn/Mul/output_0/model/layers.15/mlp/act_fn/Mul"Mul

�

(/model/layers.15/mlp/act_fn/Mul/output_0

,/model/layers.15/mlp/up_proj/MatMul/output_0!/model/layers.15/mlp/Mul/output_0/model/layers.15/mlp/Mul"Mul

�

!/model/layers.15/mlp/Mul/output_0

+model.layers.15.mlp.down_proj.MatMul.weight./model/layers.15/mlp/down_proj/MatMul/output_0%/model/layers.15/mlp/down_proj/MatMul"MatMul

�

2/model/layers.15/post_attention_layernorm/output_3

./model/layers.15/mlp/down_proj/MatMul/output_0

&model.layers.16.input_layernorm.weight)/model/layers.16/input_layernorm/output_0��)/model/layers.16/input_layernorm/output_3./model/layers.16/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.16/input_layernorm/output_0

)model.layers.16.attn.q_proj.MatMul.weight,/model/layers.16/attn/q_proj/MatMul/output_0#/model/layers.16/attn/q_proj/MatMul"MatMul

�

)/model/layers.16/input_layernorm/output_0

)model.layers.16.attn.k_proj.MatMul.weight,/model/layers.16/attn/k_proj/MatMul/output_0#/model/layers.16/attn/k_proj/MatMul"MatMul

�

)/model/layers.16/input_layernorm/output_0

)model.layers.16.attn.v_proj.MatMul.weight,/model/layers.16/attn/v_proj/MatMul/output_0#/model/layers.16/attn/v_proj/MatMul"MatMul

�

,/model/layers.16/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.16/attn/q_rotary/RotaryEmbedding/output_0./model/layers.16/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.16/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.16/attn/k_rotary/RotaryEmbedding/output_0./model/layers.16/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.16/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.16/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.16/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.16.key

past_key_values.16.value1/model/layers.16/attn/MultiHeadAttention/output_0present.16.keypresent.16.value(/model/layers.16/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.16/attn/MultiHeadAttention/output_0

)model.layers.16.attn.o_proj.MatMul.weight,/model/layers.16/attn/o_proj/MatMul/output_0#/model/layers.16/attn/o_proj/MatMul"MatMul

�

)/model/layers.16/input_layernorm/output_3

,/model/layers.16/attn/o_proj/MatMul/output_0

/model.layers.16.post_attention_layernorm.weight2/model/layers.16/post_attention_layernorm/output_0��2/model/layers.16/post_attention_layernorm/output_37/model/layers.16/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.16/post_attention_layernorm/output_0

+model.layers.16.mlp.gate_proj.MatMul.weight./model/layers.16/mlp/gate_proj/MatMul/output_0%/model/layers.16/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.16/post_attention_layernorm/output_0

)model.layers.16.mlp.up_proj.MatMul.weight,/model/layers.16/mlp/up_proj/MatMul/output_0#/model/layers.16/mlp/up_proj/MatMul"MatMul

�

./model/layers.16/mlp/gate_proj/MatMul/output_0,/model/layers.16/mlp/act_fn/Sigmoid/output_0#/model/layers.16/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.16/mlp/gate_proj/MatMul/output_0

,/model/layers.16/mlp/act_fn/Sigmoid/output_0(/model/layers.16/mlp/act_fn/Mul/output_0/model/layers.16/mlp/act_fn/Mul"Mul

�

(/model/layers.16/mlp/act_fn/Mul/output_0

,/model/layers.16/mlp/up_proj/MatMul/output_0!/model/layers.16/mlp/Mul/output_0/model/layers.16/mlp/Mul"Mul

�

!/model/layers.16/mlp/Mul/output_0

+model.layers.16.mlp.down_proj.MatMul.weight./model/layers.16/mlp/down_proj/MatMul/output_0%/model/layers.16/mlp/down_proj/MatMul"MatMul

�

2/model/layers.16/post_attention_layernorm/output_3

./model/layers.16/mlp/down_proj/MatMul/output_0

&model.layers.17.input_layernorm.weight)/model/layers.17/input_layernorm/output_0��)/model/layers.17/input_layernorm/output_3./model/layers.17/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.17/input_layernorm/output_0

)model.layers.17.attn.q_proj.MatMul.weight,/model/layers.17/attn/q_proj/MatMul/output_0#/model/layers.17/attn/q_proj/MatMul"MatMul

�

)/model/layers.17/input_layernorm/output_0

)model.layers.17.attn.k_proj.MatMul.weight,/model/layers.17/attn/k_proj/MatMul/output_0#/model/layers.17/attn/k_proj/MatMul"MatMul

�

)/model/layers.17/input_layernorm/output_0

)model.layers.17.attn.v_proj.MatMul.weight,/model/layers.17/attn/v_proj/MatMul/output_0#/model/layers.17/attn/v_proj/MatMul"MatMul

�

,/model/layers.17/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.17/attn/q_rotary/RotaryEmbedding/output_0./model/layers.17/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.17/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.17/attn/k_rotary/RotaryEmbedding/output_0./model/layers.17/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.17/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.17/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.17/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.17.key

past_key_values.17.value1/model/layers.17/attn/MultiHeadAttention/output_0present.17.keypresent.17.value(/model/layers.17/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.17/attn/MultiHeadAttention/output_0

)model.layers.17.attn.o_proj.MatMul.weight,/model/layers.17/attn/o_proj/MatMul/output_0#/model/layers.17/attn/o_proj/MatMul"MatMul

�

)/model/layers.17/input_layernorm/output_3

,/model/layers.17/attn/o_proj/MatMul/output_0

/model.layers.17.post_attention_layernorm.weight2/model/layers.17/post_attention_layernorm/output_0��2/model/layers.17/post_attention_layernorm/output_37/model/layers.17/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.17/post_attention_layernorm/output_0

+model.layers.17.mlp.gate_proj.MatMul.weight./model/layers.17/mlp/gate_proj/MatMul/output_0%/model/layers.17/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.17/post_attention_layernorm/output_0

)model.layers.17.mlp.up_proj.MatMul.weight,/model/layers.17/mlp/up_proj/MatMul/output_0#/model/layers.17/mlp/up_proj/MatMul"MatMul

�

./model/layers.17/mlp/gate_proj/MatMul/output_0,/model/layers.17/mlp/act_fn/Sigmoid/output_0#/model/layers.17/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.17/mlp/gate_proj/MatMul/output_0

,/model/layers.17/mlp/act_fn/Sigmoid/output_0(/model/layers.17/mlp/act_fn/Mul/output_0/model/layers.17/mlp/act_fn/Mul"Mul

�

(/model/layers.17/mlp/act_fn/Mul/output_0

,/model/layers.17/mlp/up_proj/MatMul/output_0!/model/layers.17/mlp/Mul/output_0/model/layers.17/mlp/Mul"Mul

�

!/model/layers.17/mlp/Mul/output_0

+model.layers.17.mlp.down_proj.MatMul.weight./model/layers.17/mlp/down_proj/MatMul/output_0%/model/layers.17/mlp/down_proj/MatMul"MatMul

�

2/model/layers.17/post_attention_layernorm/output_3

./model/layers.17/mlp/down_proj/MatMul/output_0

&model.layers.18.input_layernorm.weight)/model/layers.18/input_layernorm/output_0��)/model/layers.18/input_layernorm/output_3./model/layers.18/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.18/input_layernorm/output_0

)model.layers.18.attn.q_proj.MatMul.weight,/model/layers.18/attn/q_proj/MatMul/output_0#/model/layers.18/attn/q_proj/MatMul"MatMul

�

)/model/layers.18/input_layernorm/output_0

)model.layers.18.attn.k_proj.MatMul.weight,/model/layers.18/attn/k_proj/MatMul/output_0#/model/layers.18/attn/k_proj/MatMul"MatMul

�

)/model/layers.18/input_layernorm/output_0

)model.layers.18.attn.v_proj.MatMul.weight,/model/layers.18/attn/v_proj/MatMul/output_0#/model/layers.18/attn/v_proj/MatMul"MatMul

�

,/model/layers.18/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.18/attn/q_rotary/RotaryEmbedding/output_0./model/layers.18/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.18/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.18/attn/k_rotary/RotaryEmbedding/output_0./model/layers.18/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.18/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.18/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.18/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.18.key

past_key_values.18.value1/model/layers.18/attn/MultiHeadAttention/output_0present.18.keypresent.18.value(/model/layers.18/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.18/attn/MultiHeadAttention/output_0

)model.layers.18.attn.o_proj.MatMul.weight,/model/layers.18/attn/o_proj/MatMul/output_0#/model/layers.18/attn/o_proj/MatMul"MatMul

�

)/model/layers.18/input_layernorm/output_3

,/model/layers.18/attn/o_proj/MatMul/output_0

/model.layers.18.post_attention_layernorm.weight2/model/layers.18/post_attention_layernorm/output_0��2/model/layers.18/post_attention_layernorm/output_37/model/layers.18/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.18/post_attention_layernorm/output_0

+model.layers.18.mlp.gate_proj.MatMul.weight./model/layers.18/mlp/gate_proj/MatMul/output_0%/model/layers.18/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.18/post_attention_layernorm/output_0

)model.layers.18.mlp.up_proj.MatMul.weight,/model/layers.18/mlp/up_proj/MatMul/output_0#/model/layers.18/mlp/up_proj/MatMul"MatMul

�

./model/layers.18/mlp/gate_proj/MatMul/output_0,/model/layers.18/mlp/act_fn/Sigmoid/output_0#/model/layers.18/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.18/mlp/gate_proj/MatMul/output_0

,/model/layers.18/mlp/act_fn/Sigmoid/output_0(/model/layers.18/mlp/act_fn/Mul/output_0/model/layers.18/mlp/act_fn/Mul"Mul

�

(/model/layers.18/mlp/act_fn/Mul/output_0

,/model/layers.18/mlp/up_proj/MatMul/output_0!/model/layers.18/mlp/Mul/output_0/model/layers.18/mlp/Mul"Mul

�

!/model/layers.18/mlp/Mul/output_0

+model.layers.18.mlp.down_proj.MatMul.weight./model/layers.18/mlp/down_proj/MatMul/output_0%/model/layers.18/mlp/down_proj/MatMul"MatMul

�

2/model/layers.18/post_attention_layernorm/output_3

./model/layers.18/mlp/down_proj/MatMul/output_0

&model.layers.19.input_layernorm.weight)/model/layers.19/input_layernorm/output_0��)/model/layers.19/input_layernorm/output_3./model/layers.19/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.19/input_layernorm/output_0

)model.layers.19.attn.q_proj.MatMul.weight,/model/layers.19/attn/q_proj/MatMul/output_0#/model/layers.19/attn/q_proj/MatMul"MatMul

�

)/model/layers.19/input_layernorm/output_0

)model.layers.19.attn.k_proj.MatMul.weight,/model/layers.19/attn/k_proj/MatMul/output_0#/model/layers.19/attn/k_proj/MatMul"MatMul

�

)/model/layers.19/input_layernorm/output_0

)model.layers.19.attn.v_proj.MatMul.weight,/model/layers.19/attn/v_proj/MatMul/output_0#/model/layers.19/attn/v_proj/MatMul"MatMul

�

,/model/layers.19/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.19/attn/q_rotary/RotaryEmbedding/output_0./model/layers.19/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.19/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.19/attn/k_rotary/RotaryEmbedding/output_0./model/layers.19/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.19/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.19/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.19/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.19.key

past_key_values.19.value1/model/layers.19/attn/MultiHeadAttention/output_0present.19.keypresent.19.value(/model/layers.19/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.19/attn/MultiHeadAttention/output_0

)model.layers.19.attn.o_proj.MatMul.weight,/model/layers.19/attn/o_proj/MatMul/output_0#/model/layers.19/attn/o_proj/MatMul"MatMul

�

)/model/layers.19/input_layernorm/output_3

,/model/layers.19/attn/o_proj/MatMul/output_0

/model.layers.19.post_attention_layernorm.weight2/model/layers.19/post_attention_layernorm/output_0��2/model/layers.19/post_attention_layernorm/output_37/model/layers.19/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.19/post_attention_layernorm/output_0

+model.layers.19.mlp.gate_proj.MatMul.weight./model/layers.19/mlp/gate_proj/MatMul/output_0%/model/layers.19/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.19/post_attention_layernorm/output_0

)model.layers.19.mlp.up_proj.MatMul.weight,/model/layers.19/mlp/up_proj/MatMul/output_0#/model/layers.19/mlp/up_proj/MatMul"MatMul

�

./model/layers.19/mlp/gate_proj/MatMul/output_0,/model/layers.19/mlp/act_fn/Sigmoid/output_0#/model/layers.19/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.19/mlp/gate_proj/MatMul/output_0

,/model/layers.19/mlp/act_fn/Sigmoid/output_0(/model/layers.19/mlp/act_fn/Mul/output_0/model/layers.19/mlp/act_fn/Mul"Mul

�

(/model/layers.19/mlp/act_fn/Mul/output_0

,/model/layers.19/mlp/up_proj/MatMul/output_0!/model/layers.19/mlp/Mul/output_0/model/layers.19/mlp/Mul"Mul

�

!/model/layers.19/mlp/Mul/output_0

+model.layers.19.mlp.down_proj.MatMul.weight./model/layers.19/mlp/down_proj/MatMul/output_0%/model/layers.19/mlp/down_proj/MatMul"MatMul

�

2/model/layers.19/post_attention_layernorm/output_3

./model/layers.19/mlp/down_proj/MatMul/output_0

&model.layers.20.input_layernorm.weight)/model/layers.20/input_layernorm/output_0��)/model/layers.20/input_layernorm/output_3./model/layers.20/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.20/input_layernorm/output_0

)model.layers.20.attn.q_proj.MatMul.weight,/model/layers.20/attn/q_proj/MatMul/output_0#/model/layers.20/attn/q_proj/MatMul"MatMul

�

)/model/layers.20/input_layernorm/output_0

)model.layers.20.attn.k_proj.MatMul.weight,/model/layers.20/attn/k_proj/MatMul/output_0#/model/layers.20/attn/k_proj/MatMul"MatMul

�

)/model/layers.20/input_layernorm/output_0

)model.layers.20.attn.v_proj.MatMul.weight,/model/layers.20/attn/v_proj/MatMul/output_0#/model/layers.20/attn/v_proj/MatMul"MatMul

�

,/model/layers.20/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.20/attn/q_rotary/RotaryEmbedding/output_0./model/layers.20/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.20/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.20/attn/k_rotary/RotaryEmbedding/output_0./model/layers.20/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.20/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.20/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.20/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.20.key

past_key_values.20.value1/model/layers.20/attn/MultiHeadAttention/output_0present.20.keypresent.20.value(/model/layers.20/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.20/attn/MultiHeadAttention/output_0

)model.layers.20.attn.o_proj.MatMul.weight,/model/layers.20/attn/o_proj/MatMul/output_0#/model/layers.20/attn/o_proj/MatMul"MatMul

�

)/model/layers.20/input_layernorm/output_3

,/model/layers.20/attn/o_proj/MatMul/output_0

/model.layers.20.post_attention_layernorm.weight2/model/layers.20/post_attention_layernorm/output_0��2/model/layers.20/post_attention_layernorm/output_37/model/layers.20/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.20/post_attention_layernorm/output_0

+model.layers.20.mlp.gate_proj.MatMul.weight./model/layers.20/mlp/gate_proj/MatMul/output_0%/model/layers.20/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.20/post_attention_layernorm/output_0

)model.layers.20.mlp.up_proj.MatMul.weight,/model/layers.20/mlp/up_proj/MatMul/output_0#/model/layers.20/mlp/up_proj/MatMul"MatMul

�

./model/layers.20/mlp/gate_proj/MatMul/output_0,/model/layers.20/mlp/act_fn/Sigmoid/output_0#/model/layers.20/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.20/mlp/gate_proj/MatMul/output_0

,/model/layers.20/mlp/act_fn/Sigmoid/output_0(/model/layers.20/mlp/act_fn/Mul/output_0/model/layers.20/mlp/act_fn/Mul"Mul

�

(/model/layers.20/mlp/act_fn/Mul/output_0

,/model/layers.20/mlp/up_proj/MatMul/output_0!/model/layers.20/mlp/Mul/output_0/model/layers.20/mlp/Mul"Mul

�

!/model/layers.20/mlp/Mul/output_0

+model.layers.20.mlp.down_proj.MatMul.weight./model/layers.20/mlp/down_proj/MatMul/output_0%/model/layers.20/mlp/down_proj/MatMul"MatMul

�

2/model/layers.20/post_attention_layernorm/output_3

./model/layers.20/mlp/down_proj/MatMul/output_0

&model.layers.21.input_layernorm.weight)/model/layers.21/input_layernorm/output_0��)/model/layers.21/input_layernorm/output_3./model/layers.21/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.21/input_layernorm/output_0

)model.layers.21.attn.q_proj.MatMul.weight,/model/layers.21/attn/q_proj/MatMul/output_0#/model/layers.21/attn/q_proj/MatMul"MatMul

�

)/model/layers.21/input_layernorm/output_0

)model.layers.21.attn.k_proj.MatMul.weight,/model/layers.21/attn/k_proj/MatMul/output_0#/model/layers.21/attn/k_proj/MatMul"MatMul

�

)/model/layers.21/input_layernorm/output_0

)model.layers.21.attn.v_proj.MatMul.weight,/model/layers.21/attn/v_proj/MatMul/output_0#/model/layers.21/attn/v_proj/MatMul"MatMul

�

,/model/layers.21/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.21/attn/q_rotary/RotaryEmbedding/output_0./model/layers.21/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.21/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.21/attn/k_rotary/RotaryEmbedding/output_0./model/layers.21/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.21/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.21/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.21/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.21.key

past_key_values.21.value1/model/layers.21/attn/MultiHeadAttention/output_0present.21.keypresent.21.value(/model/layers.21/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.21/attn/MultiHeadAttention/output_0

)model.layers.21.attn.o_proj.MatMul.weight,/model/layers.21/attn/o_proj/MatMul/output_0#/model/layers.21/attn/o_proj/MatMul"MatMul

�

)/model/layers.21/input_layernorm/output_3

,/model/layers.21/attn/o_proj/MatMul/output_0

/model.layers.21.post_attention_layernorm.weight2/model/layers.21/post_attention_layernorm/output_0��2/model/layers.21/post_attention_layernorm/output_37/model/layers.21/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.21/post_attention_layernorm/output_0

+model.layers.21.mlp.gate_proj.MatMul.weight./model/layers.21/mlp/gate_proj/MatMul/output_0%/model/layers.21/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.21/post_attention_layernorm/output_0

)model.layers.21.mlp.up_proj.MatMul.weight,/model/layers.21/mlp/up_proj/MatMul/output_0#/model/layers.21/mlp/up_proj/MatMul"MatMul

�

./model/layers.21/mlp/gate_proj/MatMul/output_0,/model/layers.21/mlp/act_fn/Sigmoid/output_0#/model/layers.21/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.21/mlp/gate_proj/MatMul/output_0

,/model/layers.21/mlp/act_fn/Sigmoid/output_0(/model/layers.21/mlp/act_fn/Mul/output_0/model/layers.21/mlp/act_fn/Mul"Mul

�

(/model/layers.21/mlp/act_fn/Mul/output_0

,/model/layers.21/mlp/up_proj/MatMul/output_0!/model/layers.21/mlp/Mul/output_0/model/layers.21/mlp/Mul"Mul

�

!/model/layers.21/mlp/Mul/output_0

+model.layers.21.mlp.down_proj.MatMul.weight./model/layers.21/mlp/down_proj/MatMul/output_0%/model/layers.21/mlp/down_proj/MatMul"MatMul

�

2/model/layers.21/post_attention_layernorm/output_3

./model/layers.21/mlp/down_proj/MatMul/output_0

&model.layers.22.input_layernorm.weight)/model/layers.22/input_layernorm/output_0��)/model/layers.22/input_layernorm/output_3./model/layers.22/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.22/input_layernorm/output_0

)model.layers.22.attn.q_proj.MatMul.weight,/model/layers.22/attn/q_proj/MatMul/output_0#/model/layers.22/attn/q_proj/MatMul"MatMul

�

)/model/layers.22/input_layernorm/output_0

)model.layers.22.attn.k_proj.MatMul.weight,/model/layers.22/attn/k_proj/MatMul/output_0#/model/layers.22/attn/k_proj/MatMul"MatMul

�

)/model/layers.22/input_layernorm/output_0

)model.layers.22.attn.v_proj.MatMul.weight,/model/layers.22/attn/v_proj/MatMul/output_0#/model/layers.22/attn/v_proj/MatMul"MatMul

�

,/model/layers.22/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.22/attn/q_rotary/RotaryEmbedding/output_0./model/layers.22/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.22/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.22/attn/k_rotary/RotaryEmbedding/output_0./model/layers.22/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.22/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.22/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.22/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.22.key

past_key_values.22.value1/model/layers.22/attn/MultiHeadAttention/output_0present.22.keypresent.22.value(/model/layers.22/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.22/attn/MultiHeadAttention/output_0

)model.layers.22.attn.o_proj.MatMul.weight,/model/layers.22/attn/o_proj/MatMul/output_0#/model/layers.22/attn/o_proj/MatMul"MatMul

�

)/model/layers.22/input_layernorm/output_3

,/model/layers.22/attn/o_proj/MatMul/output_0

/model.layers.22.post_attention_layernorm.weight2/model/layers.22/post_attention_layernorm/output_0��2/model/layers.22/post_attention_layernorm/output_37/model/layers.22/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.22/post_attention_layernorm/output_0

+model.layers.22.mlp.gate_proj.MatMul.weight./model/layers.22/mlp/gate_proj/MatMul/output_0%/model/layers.22/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.22/post_attention_layernorm/output_0

)model.layers.22.mlp.up_proj.MatMul.weight,/model/layers.22/mlp/up_proj/MatMul/output_0#/model/layers.22/mlp/up_proj/MatMul"MatMul

�

./model/layers.22/mlp/gate_proj/MatMul/output_0,/model/layers.22/mlp/act_fn/Sigmoid/output_0#/model/layers.22/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.22/mlp/gate_proj/MatMul/output_0

,/model/layers.22/mlp/act_fn/Sigmoid/output_0(/model/layers.22/mlp/act_fn/Mul/output_0/model/layers.22/mlp/act_fn/Mul"Mul

�

(/model/layers.22/mlp/act_fn/Mul/output_0

,/model/layers.22/mlp/up_proj/MatMul/output_0!/model/layers.22/mlp/Mul/output_0/model/layers.22/mlp/Mul"Mul

�

!/model/layers.22/mlp/Mul/output_0

+model.layers.22.mlp.down_proj.MatMul.weight./model/layers.22/mlp/down_proj/MatMul/output_0%/model/layers.22/mlp/down_proj/MatMul"MatMul

�

2/model/layers.22/post_attention_layernorm/output_3

./model/layers.22/mlp/down_proj/MatMul/output_0

&model.layers.23.input_layernorm.weight)/model/layers.23/input_layernorm/output_0��)/model/layers.23/input_layernorm/output_3./model/layers.23/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.23/input_layernorm/output_0

)model.layers.23.attn.q_proj.MatMul.weight,/model/layers.23/attn/q_proj/MatMul/output_0#/model/layers.23/attn/q_proj/MatMul"MatMul

�

)/model/layers.23/input_layernorm/output_0

)model.layers.23.attn.k_proj.MatMul.weight,/model/layers.23/attn/k_proj/MatMul/output_0#/model/layers.23/attn/k_proj/MatMul"MatMul

�

)/model/layers.23/input_layernorm/output_0

)model.layers.23.attn.v_proj.MatMul.weight,/model/layers.23/attn/v_proj/MatMul/output_0#/model/layers.23/attn/v_proj/MatMul"MatMul

�

,/model/layers.23/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.23/attn/q_rotary/RotaryEmbedding/output_0./model/layers.23/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.23/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.23/attn/k_rotary/RotaryEmbedding/output_0./model/layers.23/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.23/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.23/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.23/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.23.key

past_key_values.23.value1/model/layers.23/attn/MultiHeadAttention/output_0present.23.keypresent.23.value(/model/layers.23/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.23/attn/MultiHeadAttention/output_0

)model.layers.23.attn.o_proj.MatMul.weight,/model/layers.23/attn/o_proj/MatMul/output_0#/model/layers.23/attn/o_proj/MatMul"MatMul

�

)/model/layers.23/input_layernorm/output_3

,/model/layers.23/attn/o_proj/MatMul/output_0

/model.layers.23.post_attention_layernorm.weight2/model/layers.23/post_attention_layernorm/output_0��2/model/layers.23/post_attention_layernorm/output_37/model/layers.23/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.23/post_attention_layernorm/output_0

+model.layers.23.mlp.gate_proj.MatMul.weight./model/layers.23/mlp/gate_proj/MatMul/output_0%/model/layers.23/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.23/post_attention_layernorm/output_0

)model.layers.23.mlp.up_proj.MatMul.weight,/model/layers.23/mlp/up_proj/MatMul/output_0#/model/layers.23/mlp/up_proj/MatMul"MatMul

�

./model/layers.23/mlp/gate_proj/MatMul/output_0,/model/layers.23/mlp/act_fn/Sigmoid/output_0#/model/layers.23/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.23/mlp/gate_proj/MatMul/output_0

,/model/layers.23/mlp/act_fn/Sigmoid/output_0(/model/layers.23/mlp/act_fn/Mul/output_0/model/layers.23/mlp/act_fn/Mul"Mul

�

(/model/layers.23/mlp/act_fn/Mul/output_0

,/model/layers.23/mlp/up_proj/MatMul/output_0!/model/layers.23/mlp/Mul/output_0/model/layers.23/mlp/Mul"Mul

�

!/model/layers.23/mlp/Mul/output_0

+model.layers.23.mlp.down_proj.MatMul.weight./model/layers.23/mlp/down_proj/MatMul/output_0%/model/layers.23/mlp/down_proj/MatMul"MatMul

�

2/model/layers.23/post_attention_layernorm/output_3

./model/layers.23/mlp/down_proj/MatMul/output_0

&model.layers.24.input_layernorm.weight)/model/layers.24/input_layernorm/output_0��)/model/layers.24/input_layernorm/output_3./model/layers.24/input_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

)/model/layers.24/input_layernorm/output_0

)model.layers.24.attn.q_proj.MatMul.weight,/model/layers.24/attn/q_proj/MatMul/output_0#/model/layers.24/attn/q_proj/MatMul"MatMul

�

)/model/layers.24/input_layernorm/output_0

)model.layers.24.attn.k_proj.MatMul.weight,/model/layers.24/attn/k_proj/MatMul/output_0#/model/layers.24/attn/k_proj/MatMul"MatMul

�

)/model/layers.24/input_layernorm/output_0

)model.layers.24.attn.v_proj.MatMul.weight,/model/layers.24/attn/v_proj/MatMul/output_0#/model/layers.24/attn/v_proj/MatMul"MatMul

�

,/model/layers.24/attn/q_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.24/attn/q_rotary/RotaryEmbedding/output_0./model/layers.24/attn/q_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

,/model/layers.24/attn/k_proj/MatMul/output_0

position_ids

cos_cache

sin_cache7/model/layers.24/attn/k_rotary/RotaryEmbedding/output_0./model/layers.24/attn/k_rotary/RotaryEmbedding"RotaryEmbedding*

interleaved��*

num_heads��*

rotary_embedding_dim��: com.microsoft

�

7/model/layers.24/attn/q_rotary/RotaryEmbedding/output_0

7/model/layers.24/attn/k_rotary/RotaryEmbedding/output_0

,/model/layers.24/attn/v_proj/MatMul/output_0

�

'/model/attn_mask_reformat/Tile/output_0

past_key_values.24.key

past_key_values.24.value1/model/layers.24/attn/MultiHeadAttention/output_0present.24.keypresent.24.value(/model/layers.24/attn/MultiHeadAttention"MultiHeadAttention*

num_heads �*

scale��=�: com.microsoft

�

1/model/layers.24/attn/MultiHeadAttention/output_0

)model.layers.24.attn.o_proj.MatMul.weight,/model/layers.24/attn/o_proj/MatMul/output_0#/model/layers.24/attn/o_proj/MatMul"MatMul

�

)/model/layers.24/input_layernorm/output_3

,/model/layers.24/attn/o_proj/MatMul/output_0

/model.layers.24.post_attention_layernorm.weight2/model/layers.24/post_attention_layernorm/output_0��2/model/layers.24/post_attention_layernorm/output_37/model/layers.24/post_attention_layernorm/SkipLayerNorm" SkipSimplifiedLayerNormalization*

epsilon��'7�: com.microsoft

�

2/model/layers.24/post_attention_layernorm/output_0

+model.layers.24.mlp.gate_proj.MatMul.weight./model/layers.24/mlp/gate_proj/MatMul/output_0%/model/layers.24/mlp/gate_proj/MatMul"MatMul

�

2/model/layers.24/post_attention_layernorm/output_0

)model.layers.24.mlp.up_proj.MatMul.weight,/model/layers.24/mlp/up_proj/MatMul/output_0#/model/layers.24/mlp/up_proj/MatMul"MatMul

�

./model/layers.24/mlp/gate_proj/MatMul/output_0,/model/layers.24/mlp/act_fn/Sigmoid/output_0#/model/layers.24/mlp/act_fn/Sigmoid"Sigmoid

�

./model/layers.24/mlp/gate_proj/MatMul/output_0

,/model/layers.24/mlp/act_fn/Sigmoid/output_0(/model/layers.24/mlp/act_fn/Mul/output_0/model/layers.24/mlp/act_fn/Mul"Mul

�

(/model/layers.24/mlp/act_fn/Mul/output_0

,/model/layers.24/mlp/up_p