-
\ No newline at end of file
+
diff --git a/docs/autodistillation.md b/docs/autodistillation.md
index b1a771a48fe..985c634ed98 100644
--- a/docs/autodistillation.md
+++ b/docs/autodistillation.md
@@ -519,4 +519,4 @@ auto_distillation:
...
```
-Please refer to [example](../examples/huggingface/pytorch/text-classification/distillation/run_glue.py) for the details
\ No newline at end of file
+Please refer to [example](../examples/huggingface/pytorch/text-classification/distillation/run_glue.py) for the details
diff --git a/docs/build_docs/source/example.rst b/docs/build_docs/source/example.rst
index 809a31b9396..727ee7e6e57 100644
--- a/docs/build_docs/source/example.rst
+++ b/docs/build_docs/source/example.rst
@@ -6,5 +6,3 @@ Example
docs/examples.md
docs/intel_extension_for_transformers/llm/runtime/deprecated/docs/validated_model.md
-
-
diff --git a/docs/build_docs/source/feature.rst b/docs/build_docs/source/feature.rst
index 7a5e4f60225..85a2e455738 100644
--- a/docs/build_docs/source/feature.rst
+++ b/docs/build_docs/source/feature.rst
@@ -15,4 +15,3 @@ Features
docs/metrics.md
docs/objectives.md
docs/pipeline.md
-
diff --git a/docs/build_docs/source/index.rst b/docs/build_docs/source/index.rst
index 8d93104b721..fe9a5b90ece 100644
--- a/docs/build_docs/source/index.rst
+++ b/docs/build_docs/source/index.rst
@@ -16,5 +16,3 @@ Sections
docs/release.md
docs/legal.md
Repo
-
\ No newline at end of file
+
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/docs/profiling.md b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/docs/profiling.md
index de3a0977d64..e3299f95af1 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/docs/profiling.md
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/docs/profiling.md
@@ -167,4 +167,3 @@ XDIS 2a5: RET BASE C3 ret
```
-
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/README.md b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/README.md
index f1530316772..ee1ffa4a3f1 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/README.md
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/README.md
@@ -12,5 +12,3 @@ You have to prepare a model.bin file and its conf.yaml file for neural engine mo
### 3. Analysis results
You could see visualization one of results as below hotmap figure after running above command in the same directory, and it could generate JPG format pictures for each weight tensor, the name of pictures corresponding with tensor name.
-
-
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/requirements.txt b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/requirements.txt
index 120bc061201..9aa3581de8b 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/requirements.txt
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/requirements.txt
@@ -1,4 +1,4 @@
matplotlib
numpy
-torch~=1.13
pyyaml
+torch~=1.13
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/sparsity_all.py b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/sparsity_all.py
index e60fb540098..a2329dba547 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/sparsity_all.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/kernels/scripts/sparsity_all.py
@@ -50,7 +50,7 @@ def cal_sparse_pytorch(tensor, block):
type=int,
default=0,
help="modeltype: 0 pytorch model, 1 nn model"
-
+
)
parser.add_argument(
'--path',
@@ -64,13 +64,13 @@ def cal_sparse_pytorch(tensor, block):
print("-"*100+"visualize pytorch model"+"-"*100)
print("model path:"+str(args.path))
model_file=''
- for root, dirs, files in os.walk(args.path):
+ for root, dirs, files in os.walk(args.path):
for file_name in files: #search .bin file
if '.bin' in file_name:
model_file=file_name
if model_file=='':
- assert "Must contain .bin file in this directory!"
- path_model=args.path+"/"+model_file
+ assert "Must contain .bin file in this directory!"
+ path_model=args.path+"/"+model_file
checkpoint=torch.load(path_model,map_location="cpu")
offset_path=args.path+"/pytorch_model_hotmaps/"
if not os.path.exists(offset_path):
@@ -81,7 +81,7 @@ def cal_sparse_pytorch(tensor, block):
for name in checkpoint:
if "weight" in name and len((checkpoint[name]).shape)>1: #and name.find("weight") > 0
tensor = checkpoint[name]
- tensor_size = int(tensor.numel()) #get number of element
+ tensor_size = int(tensor.numel()) #get number of element
elt_zero = tensor_size - tensor.nonzero().size(0)
elt_sparse = elt_zero / tensor_size
if len(tensor.size()) > 1:
@@ -106,7 +106,7 @@ def cal_sparse_pytorch(tensor, block):
print(name, elt_zero, elt_sparse, tensor.size())
f.write("{}:{},{}\t{}".format(name, elt_zero, elt_sparse, tensor.size()))
-
+
if args.modeltype==1:
print("-"*100+"visualize nn model"+"-"*100)
print("model path:"+str(args.path))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/gtest/kernels/FindISA.cmake b/intel_extension_for_transformers/llm/runtime/deprecated/test/gtest/kernels/FindISA.cmake
index 7d6745a3f98..caa8573aa76 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/gtest/kernels/FindISA.cmake
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/gtest/kernels/FindISA.cmake
@@ -47,10 +47,10 @@ endmacro()
macro(check_isa isa)
if(WIN32)
- string(TOUPPER "${isa}" UPPER_ISA)
+ string(TOUPPER "${isa}" UPPER_ISA)
check_isa_win("${UPPER_ISA}")
endif()
- if(UNIX)
+ if(UNIX)
check_isa_unix("${isa}")
endif()
endmacro()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/requirements.txt b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/requirements.txt
index e577ddd6a67..22240c8f888 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/requirements.txt
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/requirements.txt
@@ -1,8 +1,8 @@
+datasets
+intel-extension-for-pytorch==2.0.0
+intel-tensorflow==2.12.0
onnx
onnxruntime
-datasets
-transformers
onnxruntime_extensions
torch==2.0
-intel-tensorflow==2.12.0
-intel-extension-for-pytorch==2.0.0
+transformers
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_add_embeddings.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_add_embeddings.py
index cd2baccf7e8..cacdf7e76de 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_add_embeddings.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_add_embeddings.py
@@ -39,20 +39,20 @@ def test_add_embeddings_with_seq_len_first(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
add_node = OPERATORS['Add']()
input_tensors = [Tensor(name='add_src0'), Tensor(name='add_src1')]
output_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['transpose'])]
- add_node.construct('add', 'Add', input_tensors=input_tensors,
+ add_node.construct('add', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
transpose_node = OPERATORS['Transpose']()
input_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['transpose'])]
output_tensors = [Tensor(name='transpose:0', source_op=['transpose'],
dest_op=['layernorm'])]
- transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
+ transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'src_perm': '0,1,2', 'dst_perm': '1,0,2'}))
@@ -65,7 +65,7 @@ def test_add_embeddings_with_seq_len_first(self):
shape=[1024])]
output_tensors = [Tensor(name='layernorm:0', source_op=['layernorm'],
dest_op=[])]
- ln_node.construct('layernorm', 'LayerNorm', input_tensors=input_tensors,
+ ln_node.construct('layernorm', 'LayerNorm', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({'epsilon': 0.009}))
graph.insert_nodes(len(graph.nodes), [input_data_node, add_node, transpose_node, ln_node])
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_binary_op.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_binary_op.py
index aed849ebd3f..c00921289e6 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_binary_op.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_binary_op.py
@@ -70,4 +70,4 @@ def test_binary_op(self):
self.assertEqual(id_0, id_1)
if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_cast_to.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_cast_to.py
index c1280986b96..88aef522ef0 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_cast_to.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_cast_to.py
@@ -39,20 +39,20 @@ def test_softmax_cast(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
softmax_node = OPERATORS['Softmax']()
input_tensors = [Tensor(name='s_src0')]
output_tensors = [Tensor(name='softmax:0', source_op=['softmax'], dest_op=['cast'])]
- softmax_node.construct('softmax', 'Softmax', input_tensors=input_tensors,
+ softmax_node.construct('softmax', 'Softmax', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
cast_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='softmax:0', source_op=['softmax'], dest_op=['cast'])]
output_tensors = [Tensor(name='cast:0', source_op=['cast'],
dest_op=[])]
- cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
+ cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'DstT': 'fp32'}))
@@ -67,20 +67,20 @@ def test_greater_cast_reducesum(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
greater_node = OPERATORS['Greater']()
input_tensors = [Tensor(name='g_src0'), Tensor(name='g_src1',
data=np.array([1]).astype("int64"), shape=[1])]
output_tensors = [Tensor(name='greater:0', source_op=['greater'], dest_op=['cast'])]
- greater_node.construct('greater', 'Greater', input_tensors=input_tensors,
+ greater_node.construct('greater', 'Greater', input_tensors=input_tensors,
output_tensors=output_tensors)
cast_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='greater:0', source_op=['greater'], dest_op=['cast'])]
output_tensors = [Tensor(name='cast:0', source_op=['cast'], dest_op=['reducesum'])]
- cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
+ cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'DstT': 'int64'}))
@@ -88,7 +88,7 @@ def test_greater_cast_reducesum(self):
input_tensors = [Tensor(name='cast:0', source_op=['cast'], dest_op=['reducesum'])]
output_tensors = [Tensor(name='reducesum:0', source_op=['reducesum'],
dest_op=[])]
- rs_node.construct('reducesum', 'ReduceSum', input_tensors=input_tensors,
+ rs_node.construct('reducesum', 'ReduceSum', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, greater_node, cast_node, rs_node])
@@ -102,20 +102,20 @@ def test_range_cast_less(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
cast_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='c_src0')]
output_tensors = [Tensor(name='cast:0', source_op=['cast'], dest_op=['range'])]
- cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
+ cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'DstT': 'int64'}))
range_node = OPERATORS['Range']()
input_tensors = [Tensor(name='cast:0', source_op=['cast'], dest_op=['range'])]
output_tensors = [Tensor(name='range:0', source_op=['range'], dest_op=['less'])]
- range_node.construct('range', 'Range', input_tensors=input_tensors,
+ range_node.construct('range', 'Range', input_tensors=input_tensors,
output_tensors=output_tensors)
less_node = OPERATORS['Less']()
@@ -123,7 +123,7 @@ def test_range_cast_less(self):
Tensor(name='less_val', data=np.array([3]).astype("int64"))]
output_tensors = [Tensor(name='less:0', source_op=['less'],
dest_op=[])]
- less_node.construct('less', 'Less', input_tensors=input_tensors,
+ less_node.construct('less', 'Less', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, cast_node, range_node, less_node])
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_decoder_attn_reshape.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_decoder_attn_reshape.py
index e45f8957874..fcd37d8bbd7 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_decoder_attn_reshape.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_decoder_attn_reshape.py
@@ -39,54 +39,54 @@ def test_decoder_attn_reshape(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
unsqueeze1_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='unsqueeze1_src0')]
output_tensors = [Tensor(name='unsqueeze1:0', source_op=['unsqueeze1'],
dest_op=['concat'])]
- unsqueeze1_node.construct('unsqueeze1', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze1_node.construct('unsqueeze1', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
shape1_node = OPERATORS['Shape']()
input_tensors = [Tensor(name='s1_src0')]
output_tensors = [Tensor(name='shape1:0', source_op=['shape1'], dest_op=['gather1'])]
- shape1_node.construct('shape1', 'Shape', input_tensors=input_tensors,
+ shape1_node.construct('shape1', 'Shape', input_tensors=input_tensors,
output_tensors=output_tensors)
gather1_node = OPERATORS['Gather']()
input_tensors = [Tensor(name='shape1:0', source_op=['shape1'], dest_op=['gather1']),
Tensor(name='gather1_idx:0', data=np.array([0]).astype("int32"))]
output_tensors = [Tensor(name='gather1:0', source_op=['gather1'], dest_op=['unsqueeze2'])]
- gather1_node.construct('gather1', 'Gather', input_tensors=input_tensors,
+ gather1_node.construct('gather1', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors)
unsqueeze2_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='gather1:0', source_op=['gather1'], dest_op=['unsqueeze2'])]
output_tensors = [Tensor(name='unsqueeze2:0', source_op=['unsqueeze2'],
dest_op=['concat'])]
- unsqueeze2_node.construct('unsqueeze2', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze2_node.construct('unsqueeze2', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
shape2_node = OPERATORS['Shape']()
input_tensors = [Tensor(name='s2_src0')]
output_tensors = [Tensor(name='shape2:0', source_op=['shape2'], dest_op=['gather2'])]
- shape2_node.construct('shape2', 'Shape', input_tensors=input_tensors,
+ shape2_node.construct('shape2', 'Shape', input_tensors=input_tensors,
output_tensors=output_tensors)
gather2_node = OPERATORS['Gather']()
input_tensors = [Tensor(name='shape2:0', source_op=['shape2'], dest_op=['gather2']),
Tensor(name='gather2_idx:0', data=np.array([0]).astype("int32"))]
output_tensors = [Tensor(name='gather2:0', source_op=['gather2'], dest_op=['unsqueeze3'])]
- gather2_node.construct('gather2', 'Gather', input_tensors=input_tensors,
+ gather2_node.construct('gather2', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors)
unsqueeze3_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='gather2:0', source_op=['gather2'], dest_op=['unsqueeze3'])]
output_tensors = [Tensor(name='unsqueeze3:0', source_op=['unsqueeze3'],
dest_op=['concat'])]
- unsqueeze3_node.construct('unsqueeze3', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze3_node.construct('unsqueeze3', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
concat_node = OPERATORS['Concat']()
@@ -96,27 +96,27 @@ def test_decoder_attn_reshape(self):
Tensor(name='unsqueeze3:0', source_op=['unsqueeze3'], dest_op=['concat'])]
output_tensors = [Tensor(name='concat:0', source_op=['concat'],
dest_op=['reshape'])]
- concat_node.construct('concat', 'Concat', input_tensors=input_tensors,
+ concat_node.construct('concat', 'Concat', input_tensors=input_tensors,
output_tensors=output_tensors)
reshape_node = OPERATORS['Reshape']()
input_tensors = [Tensor(name='concat:0', source_op=['concat'], dest_op=['reshape'])]
output_tensors = [Tensor(name='reshape:0', source_op=['reshape'], dest_op=['gather3'])]
- reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
+ reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
gather3_node = OPERATORS['Gather']()
input_tensors = [Tensor(name='reshape:0', source_op=['reshape'], dest_op=['gather3']),
Tensor(name='gather3_idx:0', data=np.array([0]).astype("int32"))]
output_tensors = [Tensor(name='gather3:0', source_op=['gather3'], dest_op=['transpose'])]
- gather3_node.construct('gather3', 'Gather', input_tensors=input_tensors,
+ gather3_node.construct('gather3', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({'axis': 1}))
transpose_node = OPERATORS['Transpose']()
input_tensors = [Tensor(name='gather3:0', source_op=['gather3'], dest_op=['transpose'])]
output_tensors = [Tensor(name='transpose:0', source_op=['transpose'],
dest_op=[])]
- transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
+ transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, unsqueeze1_node, shape1_node,
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_embeddingbag.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_embeddingbag.py
index bf6842046ea..77bb90a0de0 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_embeddingbag.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_embeddingbag.py
@@ -32,37 +32,37 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_embeddingbag_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
squeeze_node = OPERATORS['Squeeze']()
- input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
+ input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
Tensor(data=np.array(1))]
- output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['relu'])]
- squeeze_node.construct('matmul', 'Squeeze', input_tensors=input_tensors,
+ squeeze_node.construct('matmul', 'Squeeze', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'src1_perm': '1,0'}))
-
+
embeddingbag_node = OPERATORS['EmbeddingBag']()
- input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['relu']),
- Tensor(name='matmul1:0', source_op=['matmul'],
+ Tensor(name='matmul1:0', source_op=['matmul'],
dest_op=['relu']),
- Tensor(name='matmul2:0', source_op=['matmul'],
+ Tensor(name='matmul2:0', source_op=['matmul'],
dest_op=['relu'])]
output_tensors = [Tensor(name='relu:0', source_op=['relu'],
dest_op=[])]
- embeddingbag_node.construct('relu', 'EmbeddingBag', input_tensors=input_tensors,
+ embeddingbag_node.construct('relu', 'EmbeddingBag', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, squeeze_node, embeddingbag_node])
graph = EmbeddingBag()(graph)
self.assertEqual(3, len(graph.nodes))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_execution_options.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_execution_options.py
index 047b34be526..dcda3a545a0 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_execution_options.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_execution_options.py
@@ -39,7 +39,7 @@ def test_execution_options(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(name="activation", shape=[-1, -1], dtype="fp32")]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
ip_node = OPERATORS['InnerProduct']()
input_tensors = [Tensor(name="activation", shape=[-1, -1], dtype="fp32"),
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_gelu.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_gelu.py
index 75e0431565d..b015d5d48ed 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_gelu.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_gelu.py
@@ -31,46 +31,46 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_gelu_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
div_node = OPERATORS['Div']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='div:0', source_op=['div'], dest_op=['erf'])]
- div_node.construct('div', 'Div', input_tensors=input_tensors,
+ div_node.construct('div', 'Div', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
erf_node = OPERATORS['Erf']()
input_tensors = [Tensor(name='div:0', source_op=['div'], dest_op=['erf'])]
output_tensors = [Tensor(name='erf:0', source_op=['erf'], dest_op=['add'])]
- erf_node.construct('erf', 'Erf', input_tensors=input_tensors,
+ erf_node.construct('erf', 'Erf', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
add_node = OPERATORS['Add']()
input_tensors = [Tensor(name='erf:0', source_op=['erf'], dest_op=['add'])]
output_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['mul_1'])]
- add_node.construct('add', 'Add', input_tensors=input_tensors,
+ add_node.construct('add', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
mul_1_node = OPERATORS['Mul']()
input_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['mul_1'])]
output_tensors = [Tensor(name='mul_1:0', source_op=['mul_1'], dest_op=['mul_2'])]
- mul_1_node.construct('mul_1', 'Mul', input_tensors=input_tensors,
+ mul_1_node.construct('mul_1', 'Mul', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
mul_2_node = OPERATORS['Mul']()
input_tensors = [Tensor(name='mul_1:0', source_op=['mul_1'], dest_op=['mul_2'])]
output_tensors = [Tensor(name='mul_2:0', source_op=['mul_2'], dest_op=[])]
- mul_2_node.construct('mul_2', 'Mul', input_tensors=input_tensors,
+ mul_2_node.construct('mul_2', 'Mul', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, div_node, erf_node, add_node,
mul_1_node, mul_2_node])
graph = Gelu()(graph)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_graph_dispatch.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_graph_dispatch.py
index 3ffcd68f33d..a415bbf56e5 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_graph_dispatch.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_graph_dispatch.py
@@ -42,7 +42,7 @@ def test_graph_dispatch(self):
input_0 = np.random.uniform(low=0, high=128, size=shape).astype('int32')
input_1 = np.random.uniform(low=0, high=1, size=shape).astype('int32')
input_2 = np.random.uniform(low=0, high=1, size=shape).astype('int32')
-
+
# validate int8 sparse graph tuning
int8_model_path = "/tf_dataset2/inc-ut/nlptoolkit_ut_model/bert_mini_int8_original_IR"
if is_win():
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_is_supported_onnx_graph.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_is_supported_onnx_graph.py
index 3e4df2646d2..af36efe0265 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_is_supported_onnx_graph.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_is_supported_onnx_graph.py
@@ -35,7 +35,7 @@ def tearDownClass(self):
def test_is_supported_onnx_graph(self):
'''test is_supported_onnx_graph'''
# create onnx matmul + bias graph
- # input and output
+ # input and output
a = helper.make_tensor_value_info('a', TensorProto.FLOAT, [10, 10])
x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [10, 10])
b = helper.make_tensor_value_info('b', TensorProto.FLOAT, [10, 10])
@@ -49,7 +49,7 @@ def test_is_supported_onnx_graph(self):
# test API
is_supported = is_supported_onnx_graph(graph)
self.assertEqual(is_supported, True)
-
+
'''test is_supported_onnx_node'''
ops_type = ["Add", "Softmax", "Slice", "ReduceMean", "Reshape",
"Concat", "Gather", "QuantizeLinear", "Transpose", "MatMul",
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_last_layer_shape.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_last_layer_shape.py
index 4c394811f06..eb750a1c728 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_last_layer_shape.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_last_layer_shape.py
@@ -31,47 +31,47 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_last_layer_shape_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
pack_node = OPERATORS['Pack']()
input_tensors = [Tensor(), Tensor(), Tensor(data=768)]
output_tensors = [Tensor(name='pack:0', source_op=['pack'], dest_op=['reshape'])]
- pack_node.construct('pack', 'Pack', input_tensors=input_tensors,
+ pack_node.construct('pack', 'Pack', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
reshape_node = OPERATORS['Reshape']()
input_tensors = [Tensor(name='pack:0', source_op=['pack'], dest_op=['reshape'])]
- output_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
+ output_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
dest_op=['strided_slice'])]
- reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
+ reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
strided_slice_node = OPERATORS['StridedSlice']()
- input_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
+ input_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
dest_op=['strided_slice'])]
- output_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
+ output_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['squeeze'])]
- strided_slice_node.construct('strided_slice', 'StridedSlice', input_tensors=input_tensors,
+ strided_slice_node.construct('strided_slice', 'StridedSlice', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({'test': 1}))
-
+
squeeze_node = OPERATORS['Squeeze']()
- input_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
+ input_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['squeeze'])]
output_tensors = [Tensor(name='squeeze:0', dest_op=[])]
- squeeze_node.construct('squeeze', 'Squeeze', input_tensors=input_tensors,
+ squeeze_node.construct('squeeze', 'Squeeze', input_tensors=input_tensors,
output_tensors=output_tensors)
-
- graph.insert_nodes(len(graph.nodes), [input_data_node, pack_node, reshape_node,
+
+ graph.insert_nodes(len(graph.nodes), [input_data_node, pack_node, reshape_node,
strided_slice_node, squeeze_node])
-
+
graph = LastLayerShape()(graph)
self.assertEqual(4, len(graph.nodes))
self.assertEqual('-1,-1,768', graph.nodes[1].attr['dst_shape'])
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_reduce_mean.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_reduce_mean.py
index 4cbf142ea17..ed43b9dc8ce 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_reduce_mean.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_reduce_mean.py
@@ -31,33 +31,33 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_layer_norm_with_reduce_mean_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
ln_node = OPERATORS['LayerNorm']()
input_tensors = [Tensor(), Tensor(shape=[768]), Tensor(shape=[768])]
- output_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
+ output_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
dest_op=['reduce_mean'])]
- ln_node.construct('layer_norm', 'LayerNorm', input_tensors=input_tensors,
+ ln_node.construct('layer_norm', 'LayerNorm', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'epsilon': 0.009}))
-
+
reduce_mean_node = OPERATORS['ReduceMean']()
- input_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
+ input_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
dest_op=['reduce_mean'])]
output_tensors = [Tensor(name='reduce_mean:0', source_op=['reduce_mean'],
dest_op=[])]
- reduce_mean_node.construct('reduce_mean', 'ReduceMean', input_tensors=input_tensors,
+ reduce_mean_node.construct('reduce_mean', 'ReduceMean', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict(
{'axis': 1, 'keep_dims': False}))
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, ln_node, reduce_mean_node])
graph = LayerNormWithReduceMean()(graph)
self.assertEqual(5, len(graph.nodes))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_transpose.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_transpose.py
index fc83828e6f7..e0fb037026f 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_transpose.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_layer_norm_with_transpose.py
@@ -38,31 +38,31 @@ def test_layer_norm_with_transpose(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
mat_node = OPERATORS['MatMulWithBiasAdd']()
input_tensors = [Tensor(), Tensor(shape=[768]), Tensor(shape=[768]), Tensor()]
output_tensors = [Tensor(name='mat:0', source_op=['mat'], dest_op=['layer_norm'])]
- mat_node.construct('mat', 'MatMulWithBiasAdd', input_tensors=input_tensors,
+ mat_node.construct('mat', 'MatMulWithBiasAdd', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'append_op': 'sum'}))
ln_node = OPERATORS['LayerNorm']()
input_tensors = [Tensor(name='mat:0', source_op=['mat'], dest_op=['layer_norm']),
Tensor(shape=[768]), Tensor(shape=[768])]
- output_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
+ output_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
dest_op=['transpose'])]
- ln_node.construct('layer_norm', 'LayerNorm', input_tensors=input_tensors,
+ ln_node.construct('layer_norm', 'LayerNorm', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'epsilon': 0.009}))
-
+
transpose_node = OPERATORS['Transpose']()
- input_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
+ input_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'],
dest_op=['transpose'])]
output_tensors = [Tensor(name='transpose:0', source_op=['transpose'],
dest_op=[])]
- transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
+ transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict(
{'src_perm': '0,1,2', 'dst_perm': '1,0,2'}))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_relu.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_relu.py
index cd0be0b6872..737d486dea4 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_relu.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_relu.py
@@ -32,33 +32,33 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_matmul_with_bias_relu_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
mat_node = OPERATORS['MatMulWithBias']()
- input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
+ input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
Tensor(data=np.array(1))]
- output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['relu'])]
- mat_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
+ mat_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'src1_perm': '1,0'}))
-
+
tanh_node = OPERATORS['Relu']()
- input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['relu'])]
output_tensors = [Tensor(name='relu:0', source_op=['relu'],
dest_op=[])]
- tanh_node.construct('relu', 'Relu', input_tensors=input_tensors,
+ tanh_node.construct('relu', 'Relu', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, mat_node, tanh_node])
graph = MatMulWithBiasRelu()(graph)
self.assertEqual(2, len(graph.nodes))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_sigmoid.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_sigmoid.py
index 2ad9b097a4b..b278d6e3421 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_sigmoid.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_sigmoid.py
@@ -32,33 +32,33 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_matmul_with_bias_relu_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
mat_node = OPERATORS['MatMulWithBias']()
- input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
+ input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
Tensor(data=np.array(1))]
- output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['sigmoid'])]
- mat_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
+ mat_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'src1_perm': '1,0'}))
-
+
tanh_node = OPERATORS['Sigmoid']()
- input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['sigmoid'])]
output_tensors = [Tensor(name='sigmoid:0', source_op=['sigmoid'],
dest_op=[])]
- tanh_node.construct('sigmoid', 'Sigmoid', input_tensors=input_tensors,
+ tanh_node.construct('sigmoid', 'Sigmoid', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, mat_node, tanh_node])
graph = MatMulWithBiasSigmoid()(graph)
self.assertEqual(2, len(graph.nodes))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_tanh.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_tanh.py
index 7c806213375..8176cde449a 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_tanh.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_matmul_with_bias_tanh.py
@@ -32,33 +32,33 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_matmul_with_bias_tanh_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
mat_node = OPERATORS['MatMulWithBias']()
- input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
+ input_tensors = [Tensor(data=np.array(1)), Tensor(data=np.array(1)),
Tensor(data=np.array(1))]
- output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['tanh'])]
- mat_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
+ mat_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'src1_perm': '1,0'}))
-
+
tanh_node = OPERATORS['Tanh']()
- input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
+ input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['tanh'])]
output_tensors = [Tensor(name='tanh:0', source_op=['tanh'],
dest_op=[])]
- tanh_node.construct('tanh', 'Tanh', input_tensors=input_tensors,
+ tanh_node.construct('tanh', 'Tanh', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, mat_node, tanh_node])
graph = MatmulWithBiasTanh()(graph)
self.assertEqual(2, len(graph.nodes))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_onnx_utils.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_onnx_utils.py
index 05109b645fe..bd773735bf2 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_onnx_utils.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_onnx_utils.py
@@ -35,11 +35,11 @@ def tearDownClass(self):
def test_change_num_name(self):
out = compile.onnx_utils.change_num_name(1)
self.assertEqual(1, out)
-
+
def test_change_num_namei_same(self):
out = compile.onnx_utils.change_num_name('1')
self.assertEqual('1_tensor', out)
-
+
def test_bias_to_int32_if1(self):
fake_input_tensors = [
Tensor(data=np.array([[1,2],[3,4]], dtype=np.int8)),
@@ -50,12 +50,12 @@ def test_bias_to_int32_if1(self):
Tensor(data=None),
]
fake_bias_node = OPERATORS['Add']()
- fake_bias_node.construct('bias_add', 'Add',
+ fake_bias_node.construct('bias_add', 'Add',
input_tensors=fake_input_tensors)
out = compile.onnx_utils.bias_to_int32(fake_bias_node, 0.3, 0.4)
golden_out = np.array([[1,2],[2,3]])
self.assertSequenceEqual(golden_out.tolist(), out.tolist())
-
+
def test_bias_to_int32_else(self):
fake_input_tensors = [
Tensor(data=None, source_op=[None]),
@@ -66,12 +66,12 @@ def test_bias_to_int32_else(self):
Tensor(data=np.array(0.2, dtype=np.float32)),
]
fake_bias_node = OPERATORS['Add']()
- fake_bias_node.construct('bias_add', 'Add',
+ fake_bias_node.construct('bias_add', 'Add',
input_tensors=fake_input_tensors)
out = compile.onnx_utils.bias_to_int32(fake_bias_node, 0.3, 0.4)
golden_out = np.array([[1,2],[2,3]])
self.assertSequenceEqual(golden_out.tolist(), out.tolist())
-
+
def test_bias_to_int32_if2(self):
fake_input_tensors = [
Tensor(data=np.array([[1,2],[3,4]], dtype=np.int64)),
@@ -82,7 +82,7 @@ def test_bias_to_int32_if2(self):
Tensor(data=None),
]
fake_bias_node = OPERATORS['Add']()
- fake_bias_node.construct('bias_add', 'Add',
+ fake_bias_node.construct('bias_add', 'Add',
input_tensors=fake_input_tensors)
out = compile.onnx_utils.bias_to_int32(fake_bias_node, 0.3, 0.4)
self.assertEqual(None, out)
@@ -90,7 +90,7 @@ def test_bias_to_int32_if2(self):
def test_bf16_tensor_to_array(self):
numpy_array = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float16)
print(f"Original Numpy array:\n{numpy_array}\n")
- # Convert the Numpy array to a TensorProto
+ # Convert the Numpy array to a TensorProto
tensor = onnx.helper.make_tensor('bf16_tensor', onnx.TensorProto.BFLOAT16, [2, 3], [1.0,2.0,3.0,4.0,5.0,6.0], False)
print(f"TensorProto:\n{tensor}")
out = compile.onnx_utils._bf16_tensor_to_array(tensor)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_operator_adaptor.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_operator_adaptor.py
index 4a05c5c9c12..f4ee539a3c0 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_operator_adaptor.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_operator_adaptor.py
@@ -39,14 +39,14 @@ def test_gather_sweep(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
gather1_node = OPERATORS['Gather']()
input_tensors = [Tensor(name='g1_src0'),
Tensor(name='g1_idx:0', data=np.array(0).astype("int32"))]
output_tensors = [Tensor(name='gather1:0', source_op=['gather1'], dest_op=[])]
- gather1_node.construct('gather1', 'Gather', input_tensors=input_tensors,
+ gather1_node.construct('gather1', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors)
gather1_node._filling_method = 'extract_from_onnxruntime'
@@ -54,7 +54,7 @@ def test_gather_sweep(self):
input_tensors = [Tensor(name='g2_idx:0', data=np.array(0).astype("int32")),
Tensor(name='g2_src1')]
output_tensors = [Tensor(name='gather2:0', source_op=['gather2'], dest_op=[])]
- gather2_node.construct('gather2', 'Gather', input_tensors=input_tensors,
+ gather2_node.construct('gather2', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors)
gather2_node._filling_method = 'extract_from_onnxruntime'
@@ -70,13 +70,13 @@ def test_reshape_non_2d_src_before_inner_product(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
transpose_node = OPERATORS['Transpose']()
input_tensors = [Tensor(name='t_src0')]
output_tensors = [Tensor(name='transpose:0', source_op=['transpose'], dest_op=['matmul'])]
- transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
+ transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_perm': '1,0,2'}))
@@ -84,7 +84,7 @@ def test_reshape_non_2d_src_before_inner_product(self):
input_tensors = [Tensor(name='transpose:0', source_op=['transpose'], dest_op=['matmul']),
Tensor(name='m_src1', data=np.random.randn(768, 768).astype("float32"))]
output_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=[])]
- matmul_node.construct('matmul', 'MatMul', input_tensors=input_tensors,
+ matmul_node.construct('matmul', 'MatMul', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, transpose_node, matmul_node])
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_ops.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_ops.py
index 1a2297fdc17..a3c156442c8 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_ops.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_ops.py
@@ -35,7 +35,7 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_all(self):
all_node = node_def_pb2.NodeDef()
all_node.name = 'all'
@@ -58,7 +58,7 @@ def test_assert(self):
assert_node_test.set_attr('tensorflow', assert_node)
summarize = assert_node_test.attr['summarize']
self.assertEqual(3, summarize)
-
+
def test_batch_matmul(self):
batch_matmul_node = node_def_pb2.NodeDef()
batch_matmul_node.name = 'batch_matmul'
@@ -72,7 +72,7 @@ def test_batch_matmul(self):
transpose_b = batch_matmul_node_test.attr['transpose_b']
self.assertFalse(transpose_a)
self.assertTrue(transpose_b)
-
+
def test_cast(self):
cast_node = NodeProto()
cast_node.name = 'cast'
@@ -83,7 +83,7 @@ def test_cast(self):
cast_node_test.set_attr('onnxruntime', cast_node)
dst_dtype = cast_node_test.attr['DstT']
self.assertEqual('fp32', dst_dtype)
-
+
def test_concat(self):
concat_node = NodeProto()
concat_node.name = 'concat'
@@ -94,7 +94,7 @@ def test_concat(self):
concat_node_test.set_attr('onnxruntime', concat_node)
axis = concat_node_test.attr['axis']
self.assertSequenceEqual([1, 2], axis)
-
+
def test_add(self):
add_node = NodeProto()
add_node.name = 'add'
@@ -104,7 +104,7 @@ def test_add(self):
add_node_test.name = add_node.name
name = add_node_test.name
self.assertEqual('add', name)
-
+
def test_constant_of_shape(self):
constant_of_shape_node = NodeProto()
constant_of_shape_node.name = 'constant_of_shape'
@@ -124,7 +124,7 @@ def test_dequantize_linear(self):
dequantize_linear_node_test.name = dequantize_linear_node.name
name = dequantize_linear_node_test.name
self.assertEqual('dequantize_linear', name)
-
+
def test_div(self):
div_node = NodeProto()
div_node.name = 'div'
@@ -134,7 +134,7 @@ def test_div(self):
div_node_test.name = div_node.name
name = div_node_test.name
self.assertEqual('div', name)
-
+
def test_equal(self):
equal_node = NodeProto()
equal_node.name = 'equal'
@@ -144,7 +144,7 @@ def test_equal(self):
equal_node_test.name = equal_node.name
name = equal_node_test.name
self.assertEqual('equal', name)
-
+
def test_expand(self):
expand_node = NodeProto()
expand_node.name = 'expand'
@@ -154,7 +154,7 @@ def test_expand(self):
expand_node_test.name = expand_node.name
name = expand_node_test.name
self.assertEqual('expand', name)
-
+
def test_non_zero(self):
non_zero_node = NodeProto()
non_zero_node.name = 'non_zero'
@@ -164,7 +164,7 @@ def test_non_zero(self):
non_zero_node_test.name = non_zero_node.name
name = non_zero_node_test.name
self.assertEqual('non_zero', name)
-
+
def test_qlinear_matmul(self):
qlinear_matmul_node = NodeProto()
qlinear_matmul_node.name = 'qlinear_matmul'
@@ -174,7 +174,7 @@ def test_qlinear_matmul(self):
qlinear_matmul_node_test.name = qlinear_matmul_node.name
name = qlinear_matmul_node_test.name
self.assertEqual('qlinear_matmul', name)
-
+
def test_qlinear_add(self):
qlinear_add_node = NodeProto()
qlinear_add_node.name = 'qlinear_add'
@@ -184,7 +184,7 @@ def test_qlinear_add(self):
qlinear_add_node_test.name = qlinear_add_node.name
name = qlinear_add_node_test.name
self.assertEqual('qlinear_add', name)
-
+
def test_qlinear_mul(self):
qlinear_mul_node = NodeProto()
qlinear_mul_node.name = 'qlinear_mul'
@@ -194,7 +194,7 @@ def test_qlinear_mul(self):
qlinear_mul_node_test.name = qlinear_mul_node.name
name = qlinear_mul_node_test.name
self.assertEqual('qlinear_mul', name)
-
+
def test_where(self):
where_node = NodeProto()
where_node.name = 'where'
@@ -214,7 +214,7 @@ def test_erf(self):
erf_node_test.name = erf_node.name
name = erf_node_test.name
self.assertEqual('erf', name)
-
+
def test_fill(self):
fill_node = node_def_pb2.NodeDef()
fill_node.name = 'fill'
@@ -224,7 +224,7 @@ def test_fill(self):
fill_node_test.name = fill_node.name
name = fill_node_test.name
self.assertEqual('fill', name)
-
+
def test_flat_map_dataset(self):
flat_map_dataset_node = node_def_pb2.NodeDef()
flat_map_dataset_node.name = 'flat_map_dataset'
@@ -234,7 +234,7 @@ def test_flat_map_dataset(self):
flat_map_dataset_node_test.name = flat_map_dataset_node.name
name = flat_map_dataset_node_test.name
self.assertEqual('flat_map_dataset', name)
-
+
def test_identity(self):
identity_node = node_def_pb2.NodeDef()
identity_node.name = 'identity'
@@ -244,7 +244,7 @@ def test_identity(self):
identity_node_test.name = identity_node.name
name = identity_node_test.name
self.assertEqual('identity', name)
-
+
def test_innerproduct(self):
innerproduct_node = node_def_pb2.NodeDef()
innerproduct_node.name = 'innerproduct'
@@ -254,7 +254,7 @@ def test_innerproduct(self):
innerproduct_node_test.name = innerproduct_node.name
name = innerproduct_node_test.name
self.assertEqual('innerproduct', name)
-
+
def test_less_equal(self):
less_equal_node = node_def_pb2.NodeDef()
less_equal_node.name = 'less_equal'
@@ -264,7 +264,7 @@ def test_less_equal(self):
less_equal_node_test.name = less_equal_node.name
name = less_equal_node_test.name
self.assertEqual('less_equal', name)
-
+
def test_make_iterator(self):
make_iterator_node = node_def_pb2.NodeDef()
make_iterator_node.name = 'make_iterator'
@@ -284,7 +284,7 @@ def test_matmul_with_bias_tanh(self):
matmul_with_bias_tanh_node_test.name = matmul_with_bias_tanh_node.name
name = matmul_with_bias_tanh_node_test.name
self.assertEqual('matmul_with_bias_tanh', name)
-
+
def test_pow(self):
pow_node = node_def_pb2.NodeDef()
pow_node.name = 'pow'
@@ -294,7 +294,7 @@ def test_pow(self):
pow_node_test.name = pow_node.name
name = pow_node_test.name
self.assertEqual('pow', name)
-
+
def test_real_div(self):
real_div_node = node_def_pb2.NodeDef()
real_div_node.name = 'real_div'
@@ -304,7 +304,7 @@ def test_real_div(self):
real_div_node_test.name = real_div_node.name
name = real_div_node_test.name
self.assertEqual('real_div', name)
-
+
def test_sqrt(self):
sqrt_node = node_def_pb2.NodeDef()
sqrt_node.name = 'sqrt'
@@ -314,7 +314,7 @@ def test_sqrt(self):
sqrt_node_test.name = sqrt_node.name
name = sqrt_node_test.name
self.assertEqual('sqrt', name)
-
+
def test_square(self):
square_node = node_def_pb2.NodeDef()
square_node.name = 'square'
@@ -324,7 +324,7 @@ def test_square(self):
square_node_test.name = square_node.name
name = square_node_test.name
self.assertEqual('square', name)
-
+
def test_stop_gradient(self):
stop_gradient_node = node_def_pb2.NodeDef()
stop_gradient_node.name = 'stop_gradient'
@@ -334,7 +334,7 @@ def test_stop_gradient(self):
stop_gradient_node_test.name = stop_gradient_node.name
name = stop_gradient_node_test.name
self.assertEqual('stop_gradient', name)
-
+
def test_tanh(self):
tanh_node = node_def_pb2.NodeDef()
tanh_node.name = 'tanh'
@@ -344,7 +344,7 @@ def test_tanh(self):
tanh_node_test.name = tanh_node.name
name = tanh_node_test.name
self.assertEqual('tanh', name)
-
+
def test_tensor_slice_dataset(self):
tensor_slice_dataset_node = node_def_pb2.NodeDef()
tensor_slice_dataset_node.name = 'tensor_slice_dataset'
@@ -354,7 +354,7 @@ def test_tensor_slice_dataset(self):
tensor_slice_dataset_node_test.name = tensor_slice_dataset_node.name
name = tensor_slice_dataset_node_test.name
self.assertEqual('tensor_slice_dataset', name)
-
+
def test_fused_batch_matmul_v2(self):
fused_batch_matmul_v2_node = node_def_pb2.NodeDef()
fused_batch_matmul_v2_node.name = 'fused_batch_matmul_v2'
@@ -368,7 +368,7 @@ def test_fused_batch_matmul_v2(self):
adj_y = fused_batch_matmul_v2_node_test.attr['transpose_b']
self.assertFalse(adj_x)
self.assertTrue(adj_y)
-
+
def test_fused_batch_norm_v3(self):
fused_batch_norm_v3_node = node_def_pb2.NodeDef()
fused_batch_norm_v3_node.name = 'fused_batch_norm_v3'
@@ -387,7 +387,7 @@ def test_fused_batch_norm_v3(self):
self.assertEqual(0.0010000000474974513, epsilon)
self.assertEqual(1, exponential_avg_factor)
self.assertTrue(is_training)
-
+
def test_fused_gemm(self):
fused_gemm_node = NodeProto()
fused_gemm_node.name = 'fused_gemm'
@@ -412,7 +412,7 @@ def test_fused_gemm(self):
self.assertEqual(2, alpha)
self.assertEqual(2, beta)
self.assertEqual('InnerProduct', op_type)
-
+
def test_gemm(self):
gemm_node = NodeProto()
gemm_node.name = 'gemm'
@@ -434,12 +434,12 @@ def test_gemm(self):
self.assertEqual(2, alpha)
self.assertEqual(2, beta)
self.assertEqual('MatMulWithBias', op_type)
-
+
def test_quantize_linear(self):
quantize_linear_node = NodeProto()
quantize_linear_node.name = 'quantize_linear'
quantize_linear_node.op_type = 'QuantizeLinear'
-
+
quantize_linear_node_test = OPERATORS['QuantizeLinear']()
quantize_linear_node_test.set_attr('onnxruntime', quantize_linear_node)
output_dtype = quantize_linear_node_test.attr['output_dtype']
@@ -465,7 +465,7 @@ def test_fused_matmul_onnx(self):
self.assertTrue(transpose_a)
self.assertFalse(transpose_b)
self.assertEqual(0.125, alpha)
-
+
def test_fused_matmul_tensorflow(self):
fused_matmul_node = node_def_pb2.NodeDef()
fused_matmul_node.name = 'fused_matmul'
@@ -480,7 +480,7 @@ def test_fused_matmul_tensorflow(self):
src1_perm = fused_matmul_node_test.attr['src1_perm']
self.assertEqual('1,0', src0_perm)
self.assertEqual('0,1', src1_perm)
-
+
def test_gather_onnx(self):
gather_node = NodeProto()
gather_node.name = 'gather'
@@ -495,7 +495,7 @@ def test_gather_onnx(self):
axis = gather_node_test.attr['axis']
self.assertEqual(0, batch_dims)
self.assertEqual(0, axis)
-
+
def test_gather_tensorflow(self):
gather_node = node_def_pb2.NodeDef()
gather_node.name = 'gather'
@@ -508,7 +508,7 @@ def test_gather_tensorflow(self):
axis = gather_node_test.attr['axis']
self.assertEqual(0, batch_dims)
self.assertEqual(0, axis)
-
+
def test_reduce_mean_onnx(self):
reduce_mean_node = NodeProto()
reduce_mean_node.name = 'reduce_mean'
@@ -522,7 +522,7 @@ def test_reduce_mean_onnx(self):
axis = reduce_mean_node_test.attr['axis']
self.assertFalse(keep_dims)
self.assertEqual(0, axis)
-
+
def test_reduce_mean_tensorflow(self):
reduce_mean_node = node_def_pb2.NodeDef()
reduce_mean_node.name = 'reduce_mean'
@@ -536,7 +536,7 @@ def test_reduce_mean_tensorflow(self):
axis = reduce_mean_node_test.attr['axis']
self.assertFalse(keep_dims)
self.assertEqual(0, axis)
-
+
def test_squeeze_onnx(self):
squeeze_node = NodeProto()
squeeze_node.name = 'squeeze'
@@ -547,7 +547,7 @@ def test_squeeze_onnx(self):
squeeze_node_test.set_attr('onnxruntime', squeeze_node)
axis = squeeze_node_test.attr['axis']
self.assertEqual('0,1,2', axis)
-
+
def test_squeeze_tensorflow(self):
squeeze_node = node_def_pb2.NodeDef()
squeeze_node.name = 'squeeze'
@@ -571,7 +571,7 @@ def test_iter_dataset_related(self):
attr = {'output_shapes': a(b(shape_list)),
'output_types': a(e([3]))}
fake_node = f(attr)
-
+
op_type_list = ['IteratorGetNext', 'IteratorV2', 'OptimizeDataset', 'MapAndBatchDataset']
for op_type in op_type_list:
iterator_get_next_node_test = OPERATORS[op_type]()
@@ -581,7 +581,7 @@ def test_iter_dataset_related(self):
output_types = iterator_get_next_node_test.attr['output_types']
self.assertSequenceEqual([[-1, 128]], output_shapes)
self.assertSequenceEqual(['int32'], output_types)
-
+
def test_quantize_v2(self):
quantize_v2_node = node_def_pb2.NodeDef()
quantize_v2_node.name = 'quantize_v2'
@@ -593,7 +593,7 @@ def test_quantize_v2(self):
op_type = quantize_v2_node_test.op_type
self.assertEqual('u8', output_dtype)
self.assertEqual('Quantize', op_type)
-
+
def test_quantized_fused_matmul_and_dequantize(self):
quantized_fused_matmul_and_dequantize_node = node_def_pb2.NodeDef()
quantized_fused_matmul_and_dequantize_node.name = 'quantized_fused_matmul_and_dequantize'
@@ -610,7 +610,7 @@ def test_quantized_fused_matmul_and_dequantize(self):
quantized_fused_matmul_and_dequantize_node_test = OPERATORS[
'_QuantizedFusedMatMulAndDequantize']()
- quantized_fused_matmul_and_dequantize_node_test.set_attr('tensorflow',
+ quantized_fused_matmul_and_dequantize_node_test.set_attr('tensorflow',
quantized_fused_matmul_and_dequantize_node)
src0_perm = quantized_fused_matmul_and_dequantize_node_test.attr['src0_perm']
src1_perm = quantized_fused_matmul_and_dequantize_node_test.attr['src1_perm']
@@ -622,7 +622,7 @@ def test_quantized_fused_matmul_and_dequantize(self):
self.assertEqual(1.0, epsilon)
self.assertEqual(['add'], fused_ops)
self.assertEqual('fp32', output_dtype)
-
+
def test_quantized_matmul_with_bias_and_dequantize(self):
quantized_matmul_with_bias_and_dequantize_node = node_def_pb2.NodeDef()
quantized_matmul_with_bias_and_dequantize_node.name = \
@@ -632,11 +632,11 @@ def test_quantized_matmul_with_bias_and_dequantize(self):
attr_value_pb2.AttrValue(b=True))
quantized_matmul_with_bias_and_dequantize_node.attr['transpose_b'].CopyFrom(
attr_value_pb2.AttrValue(b=False))
-
+
quantized_matmul_with_bias_and_dequantize_node_test = OPERATORS[
'QuantizedMatMulWithBiasAndDequantize']()
- quantized_matmul_with_bias_and_dequantize_node_test.set_attr('tensorflow',
+ quantized_matmul_with_bias_and_dequantize_node_test.set_attr('tensorflow',
quantized_matmul_with_bias_and_dequantize_node)
transpose_a = quantized_matmul_with_bias_and_dequantize_node_test.attr['transpose_a']
transpose_b = quantized_matmul_with_bias_and_dequantize_node_test.attr['transpose_b']
@@ -652,7 +652,7 @@ def test_layer_normalization(self):
layer_normalization_node.attribute.append(make_attribute('stash_type', 1))
layer_normalization_node.attribute.append(make_attribute('axis', 3))
layer_normalization_node.attribute.append(make_attribute('epsilon', 1.0))
-
+
op_type_list = ['LayerNormalization', '_MklLayerNorm']
for op_type in op_type_list:
layer_normalization_node_test = OPERATORS[op_type]()
@@ -663,7 +663,7 @@ def test_layer_normalization(self):
self.assertEqual(3, axis)
self.assertEqual(1.0, epsilon)
self.assertEqual('LayerNorm', op_type)
-
+
def test_one_hot(self):
one_hot_node = node_def_pb2.NodeDef()
one_hot_node.name = 'one_hot'
@@ -683,7 +683,7 @@ def test_one_hot(self):
self.assertEqual(1, on_value)
self.assertEqual(2, off_value)
self.assertEqual(1, len(one_hot_node_test.input_tensors))
-
+
def test_onnx_input(self):
a = namedtuple('fake_node', ['name', 'type'])
b = namedtuple('type', ['tensor_type'])
@@ -720,7 +720,7 @@ def test_transpose(self):
dst_perm = transpose_node_test.attr['dst_perm']
self.assertEqual('0,1,2,3', src_perm)
self.assertEqual('0,2,3,1', dst_perm)
-
+
def test_unpack(self):
unpack_node = node_def_pb2.NodeDef()
unpack_node.name = 'unpack'
@@ -734,7 +734,7 @@ def test_unpack(self):
num = unpack_node_test.attr['num']
self.assertEqual(0, axis)
self.assertEqual(2, num)
-
+
def test_unsqueeze(self):
unsqueeze_node = NodeProto()
unsqueeze_node.name = 'unsqueeze'
@@ -745,7 +745,7 @@ def test_unsqueeze(self):
unsqueeze_node_test.set_attr('onnxruntime', unsqueeze_node)
axes = unsqueeze_node_test.attr['axes']
self.assertEqual('0,2,3,1', axes)
-
+
def test_range(self):
range_node = NodeProto()
range_node.name = 'range'
@@ -755,7 +755,7 @@ def test_range(self):
range_node_test.name = range_node.name
name = range_node_test.name
self.assertEqual('range', name)
-
+
def test_relu(self):
relu_node = NodeProto()
relu_node.name = 'relu'
@@ -765,7 +765,7 @@ def test_relu(self):
relu_node_test.name = relu_node.name
name = relu_node_test.name
self.assertEqual('relu', name)
-
+
def test_matmul_with_bias_relu(self):
mat_node = NodeProto()
mat_node.name = 'matmul_with_bias_relu'
@@ -775,7 +775,7 @@ def test_matmul_with_bias_relu(self):
mat_node_test.name = mat_node.name
name = mat_node_test.name
self.assertEqual('matmul_with_bias_relu', name)
-
+
def test_matmul(self):
mat_node = NodeProto()
mat_node.name = 'matmul'
@@ -785,7 +785,7 @@ def test_matmul(self):
mat_node_test.name = mat_node.name
name = mat_node_test.name
self.assertEqual('matmul', name)
-
+
def test_quantize(self):
qat_node = NodeProto()
qat_node.name = 'quantize'
@@ -795,7 +795,7 @@ def test_quantize(self):
qat_node_test.name = qat_node.name
name = qat_node_test.name
self.assertEqual('quantize', name)
-
+
def test_not(self):
not_node = NodeProto()
not_node.name = 'not'
@@ -805,7 +805,7 @@ def test_not(self):
not_node_test.name = not_node.name
name = not_node_test.name
self.assertEqual('not', name)
-
+
def test_cumsum(self):
cumsum_node = NodeProto()
cumsum_node.name = 'cumsum'
@@ -815,7 +815,7 @@ def test_cumsum(self):
cumsum_node_test.name = cumsum_node.name
name = cumsum_node_test.name
self.assertEqual('cumsum', name)
-
+
def test_onehot(self):
onehot_node = NodeProto()
onehot_node.name = 'onehot'
@@ -825,7 +825,7 @@ def test_onehot(self):
onehot_node_test.name = onehot_node.name
name = onehot_node_test.name
self.assertEqual('onehot', name)
-
+
def test_toke_type_ids(self):
token_node = NodeProto()
token_node.name = 'toke_type_ids'
@@ -835,7 +835,7 @@ def test_toke_type_ids(self):
token_node_test.name = token_node.name
name = token_node_test.name
self.assertEqual('toke_type_ids', name)
-
+
def test_positio_ids(self):
position_node = NodeProto()
position_node.name = 'position_ids'
@@ -845,7 +845,7 @@ def test_positio_ids(self):
position_node_test.name = position_node.name
name = position_node_test.name
self.assertEqual('position_ids', name)
-
+
def test_loop(self):
loop_node = NodeProto()
loop_node.name = 'loop'
@@ -855,7 +855,7 @@ def test_loop(self):
loop_node_test.name = loop_node.name
name = loop_node_test.name
self.assertEqual('loop', name)
-
+
def test_sigmoid(self):
sigmoid_node = NodeProto()
sigmoid_node.name = 'sigmoid'
@@ -865,7 +865,7 @@ def test_sigmoid(self):
sigmoid_node_test.name = sigmoid_node.name
name = sigmoid_node_test.name
self.assertEqual('sigmoid', name)
-
+
def test_matmul_with_bias_sigmoid(self):
mat_node = NodeProto()
mat_node.name = 'matmul_with_bias_sigmoid'
@@ -875,7 +875,7 @@ def test_matmul_with_bias_sigmoid(self):
mat_node_test.name = mat_node.name
name = mat_node_test.name
self.assertEqual('matmul_with_bias_sigmoid', name)
-
+
def test_embedding_bag(self):
eb_node = NodeProto()
eb_node.name = 'embedding_bag'
@@ -885,7 +885,7 @@ def test_embedding_bag(self):
eb_node_test.name = eb_node.name
name = eb_node_test.name
self.assertEqual('embedding_bag', name)
-
+
def test_flatten(self):
flatten_node = NodeProto()
flatten_node.name = 'flatten'
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_padding_sequence.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_padding_sequence.py
index a313436b971..378b848cc38 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_padding_sequence.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_padding_sequence.py
@@ -31,86 +31,86 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_padding_sequence_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
shape_node = OPERATORS['Shape']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='shape:0', source_op=['shape'],
dest_op=['strided_slice'])]
- shape_node.construct('shape', 'Shape', input_tensors=input_tensors,
+ shape_node.construct('shape', 'Shape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
strided_slice_node = OPERATORS['StridedSlice']()
input_tensors = [Tensor(name='shape:0', source_op=['shape'],
dest_op=['strided_slice'])]
output_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['pack_0', 'pack_1'])]
- strided_slice_node.construct('strided_slice', 'StridedSlice', input_tensors=input_tensors,
+ strided_slice_node.construct('strided_slice', 'StridedSlice', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
pack_0_node = OPERATORS['Pack']()
input_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['pack_0'])]
output_tensors = [Tensor(name='pack_0:0', source_op=['pack_0'],
dest_op=['fill'])]
- pack_0_node.construct('pack_0', 'Pack', input_tensors=input_tensors,
+ pack_0_node.construct('pack_0', 'Pack', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
fill_node = OPERATORS['Fill']()
input_tensors = [Tensor(name='pack_0:0', source_op=['pack_0'],
dest_op=['fill'])]
output_tensors = [Tensor(name='fill:0', source_op=['fill'],
dest_op=['mul_1'])]
- fill_node.construct('fill', 'Fill', input_tensors=input_tensors,
+ fill_node.construct('fill', 'Fill', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
pack_1_node = OPERATORS['Pack']()
input_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['pack_1'])]
output_tensors = [Tensor(name='pack_1:0', source_op=['pack_1'],
dest_op=['reshape'])]
- pack_1_node.construct('pack_1', 'Pack', input_tensors=input_tensors,
+ pack_1_node.construct('pack_1', 'Pack', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
reshape_node = OPERATORS['Reshape']()
input_tensors = [Tensor(name='pack_1:0', source_op=['pack_1'],
dest_op=['reshape'])]
output_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
dest_op=['cast'])]
- reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
+ reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
cast_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
dest_op=['cast'])]
output_tensors = [Tensor(name='cast:0', source_op=['cast'],
dest_op=['mul_1'])]
- cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
+ cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
mul_1_node = OPERATORS['Mul']()
input_tensors = [Tensor(name='fill:0', source_op=['fill'],
dest_op=['mul_1']), Tensor(name='cast:0', source_op=['cast'],
dest_op=['mul_1'])]
output_tensors = [Tensor(name='mul_1:0', source_op=['mul_1'],
dest_op=['expand_dims'])]
- mul_1_node.construct('mul_1', 'Mul', input_tensors=input_tensors,
+ mul_1_node.construct('mul_1', 'Mul', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
expand_dims_node = OPERATORS['ExpandDims']()
input_tensors = [Tensor(name='mul_1:0', source_op=['mul_1'],
dest_op=['expand_dims'])]
output_tensors = [Tensor(name='expand_dims:0', source_op=['expand_dims'],
dest_op=['sub'])]
- expand_dims_node.construct('expand_dims', 'ExpandDims', input_tensors=input_tensors,
+ expand_dims_node.construct('expand_dims', 'ExpandDims', input_tensors=input_tensors,
output_tensors=output_tensors)
sub_node = OPERATORS['Sub']()
@@ -118,7 +118,7 @@ def test_padding_sequence_1(self):
dest_op=['sub'])]
output_tensors = [Tensor(name='sub:0', source_op=['sub'],
dest_op=['mul_2'])]
- sub_node.construct('sub', 'Sub', input_tensors=input_tensors,
+ sub_node.construct('sub', 'Sub', input_tensors=input_tensors,
output_tensors=output_tensors)
mul_2_node = OPERATORS['Mul']()
@@ -126,16 +126,16 @@ def test_padding_sequence_1(self):
dest_op=['mul_2'])]
output_tensors = [Tensor(name='mul_2:0', source_op=['mul_2'],
dest_op=['add'])]
- mul_2_node.construct('mul_2', 'Mul', input_tensors=input_tensors,
- output_tensors=output_tensors)
-
+ mul_2_node.construct('mul_2', 'Mul', input_tensors=input_tensors,
+ output_tensors=output_tensors)
+
add_node = OPERATORS['Add']()
input_tensors = [Tensor(name='mul_2:0', source_op=['mul_2'],
dest_op=['add'])]
output_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=[])]
- add_node.construct('add', 'Add', input_tensors=input_tensors,
- output_tensors=output_tensors)
-
+ add_node.construct('add', 'Add', input_tensors=input_tensors,
+ output_tensors=output_tensors)
+
graph.insert_nodes(len(graph.nodes), [input_data_node, shape_node, strided_slice_node,
pack_0_node, fill_node, pack_1_node, reshape_node,
cast_node, mul_1_node, expand_dims_node, sub_node,
@@ -152,83 +152,83 @@ def test_padding_sequence_2(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
unsqueeze_1_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='unsqueeze_1:0', source_op=['unsqueeze_1'],
dest_op=['unsqueeze_2'])]
- unsqueeze_1_node.construct('unsqueeze_1', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze_1_node.construct('unsqueeze_1', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
unsqueeze_2_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='unsqueeze_1:0', source_op=['unsqueeze_1'],
dest_op=['unsqueeze_2'])]
output_tensors = [Tensor(name='unsqueeze_2:0', source_op=['unsqueeze_2'],
dest_op=['cast'])]
- unsqueeze_2_node.construct('unsqueeze_2', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze_2_node.construct('unsqueeze_2', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
cast_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='unsqueeze_2:0', source_op=['unsqueeze_2'],
dest_op=['cast'])]
output_tensors = [Tensor(name='cast:0', source_op=['cast'],
dest_op=['sub'])]
- cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
+ cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
sub_node = OPERATORS['Sub']()
input_tensors = [Tensor(name='cast:0', source_op=['cast'],
dest_op=['sub'])]
output_tensors = [Tensor(name='sub:0', source_op=['sub'],
dest_op=['mul'])]
- sub_node.construct('sub', 'Sub', input_tensors=input_tensors,
+ sub_node.construct('sub', 'Sub', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
mul_node = OPERATORS['Mul']()
input_tensors = [Tensor(name='sub:0', source_op=['sub'],
dest_op=['mul'])]
output_tensors = [Tensor(name='mul:0', source_op=['mul'],
dest_op=['add'])]
- mul_node.construct('mul', 'Mul', input_tensors=input_tensors,
+ mul_node.construct('mul', 'Mul', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
add_node = OPERATORS['Add']()
input_tensors = [Tensor(name='mul:0', source_op=['mul'],
dest_op=['add']), Tensor(data=np.array(1))]
output_tensors = [Tensor(name='add:0', source_op=['add'],
dest_op=[])]
- add_node.construct('add', 'Add', input_tensors=input_tensors,
+ add_node.construct('add', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
mat_node = OPERATORS['MatMul']()
- input_tensors = [Tensor(),
+ input_tensors = [Tensor(),
Tensor(name='src:0', dest_op=['matmul'], shape=[768])]
output_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['add_1'])]
- mat_node.construct('matmul', 'MatMul', input_tensors=input_tensors,
+ mat_node.construct('matmul', 'MatMul', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
add_1_node = OPERATORS['Add']()
input_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['add_1']),
Tensor(data=np.array(1))]
output_tensors = [Tensor(name='add_1:0', source_op=['add_1'], dest_op=['add_2'])]
- add_1_node.construct('add_1', 'Add', input_tensors=input_tensors,
+ add_1_node.construct('add_1', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
add_2_node = OPERATORS['Add']()
input_tensors = [Tensor(name='add_1:0', source_op=['add_1'], dest_op=['add_2']),
Tensor(data=np.array(1))]
output_tensors = [Tensor(name='add_2:0', source_op=['add_2'], dest_op=['layernorm'])]
- add_2_node.construct('add_2', 'Add', input_tensors=input_tensors,
+ add_2_node.construct('add_2', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
layernorm_node = OPERATORS['LayerNorm']()
input_tensors = [Tensor(name='add_2:0', source_op=['add_2'], dest_op=['layernorm']),
- Tensor(data=np.array(1), shape=[768, 768]),
+ Tensor(data=np.array(1), shape=[768, 768]),
Tensor(data=np.array(1), shape=[768])]
output_tensors = [Tensor(name='layernorm:0', source_op=['layernorm'])]
- layernorm_node.construct('layernorm', 'LayerNorm', input_tensors=input_tensors,
+ layernorm_node.construct('layernorm', 'LayerNorm', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, unsqueeze_1_node, unsqueeze_2_node,
@@ -237,7 +237,7 @@ def test_padding_sequence_2(self):
graph = PaddingSequence()(graph)
self.assertEqual(7, len(graph.nodes))
self.assertEqual('-1,12,0,-1', graph.nodes[1].attr['dst_shape'])
-
+
def test_padding_sequence_3(self):
graph = Graph()
@@ -245,86 +245,86 @@ def test_padding_sequence_3(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
equal_node = OPERATORS['Equal']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='equal:0', source_op=['equal'],
dest_op=['reshape'])]
- equal_node.construct('equal', 'Equal', input_tensors=input_tensors,
+ equal_node.construct('equal', 'Equal', input_tensors=input_tensors,
output_tensors=output_tensors)
shape_0_node = OPERATORS['Shape']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='shape_0:0', source_op=['shape_0'],
dest_op=['gather'])]
- shape_0_node.construct('shape_0', 'Shape', input_tensors=input_tensors,
+ shape_0_node.construct('shape_0', 'Shape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
gather_node = OPERATORS['Gather']()
input_tensors = [Tensor(name='shape_0:0', source_op=['shape_0'],
dest_op=['gather'])]
output_tensors = [Tensor(name='gather:0', source_op=['gather'],
dest_op=['unsqueeze_2'])]
- gather_node.construct('gather', 'Gather', input_tensors=input_tensors,
+ gather_node.construct('gather', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
unsqueeze_1_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='unsqueeze_1:0', source_op=['unsqueeze_1'],
dest_op=['concat'])]
- unsqueeze_1_node.construct('unsqueeze_1', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze_1_node.construct('unsqueeze_1', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
unsqueeze_2_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='gather:0', source_op=['gather'],
dest_op=['unsqueeze_2'])]
output_tensors = [Tensor(name='unsqueeze_2:0', source_op=['unsqueeze_2'],
dest_op=['concat'])]
- unsqueeze_2_node.construct('unsqueeze_2', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze_2_node.construct('unsqueeze_2', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
concat_node = OPERATORS['Concat']()
input_tensors = [Tensor(name='unsqueeze_1:0', source_op=['unsqueeze_1'],
- dest_op=['concat']), Tensor(name='unsqueeze_2:0',
+ dest_op=['concat']), Tensor(name='unsqueeze_2:0',
source_op=['unsqueeze_2'], dest_op=['concat'])]
output_tensors = [Tensor(name='concat:0', source_op=['concat'],
dest_op=['reshape'])]
- concat_node.construct('concat', 'Concat', input_tensors=input_tensors,
+ concat_node.construct('concat', 'Concat', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
reshape_node = OPERATORS['Reshape']()
input_tensors = [Tensor(name='equal:0', source_op=['equal'],
- dest_op=['reshape']), Tensor(name='concat:0',
+ dest_op=['reshape']), Tensor(name='concat:0',
source_op=['concat'], dest_op=['reshape'])]
output_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
dest_op=['expand'])]
- reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
+ reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
shape_1_node = OPERATORS['Shape']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='shape_1:0', source_op=['shape_1'],
dest_op=['expand'])]
- shape_1_node.construct('shape_1', 'Shape', input_tensors=input_tensors,
+ shape_1_node.construct('shape_1', 'Shape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
expand_node = OPERATORS['Expand']()
input_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
- dest_op=['expand']), Tensor(name='shape_1:0',
+ dest_op=['expand']), Tensor(name='shape_1:0',
source_op=['shape_1'], dest_op=['expand'])]
output_tensors = [Tensor(name='expand:0', source_op=['expand'],
dest_op=['cast'])]
- expand_node.construct('expand', 'Expand', input_tensors=input_tensors,
+ expand_node.construct('expand', 'Expand', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
cast_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='expand:0', source_op=['expand'],
dest_op=['cast'])]
output_tensors = [Tensor(name='cast:0', source_op=['cast'],
dest_op=['where'])]
- cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
+ cast_node.construct('cast', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors)
where_node = OPERATORS['Where']()
@@ -332,12 +332,12 @@ def test_padding_sequence_3(self):
dest_op=['where'])]
output_tensors = [Tensor(name='where:0', source_op=['where'],
dest_op=[])]
- where_node.construct('where', 'Where', input_tensors=input_tensors,
+ where_node.construct('where', 'Where', input_tensors=input_tensors,
output_tensors=output_tensors)
-
- graph.insert_nodes(len(graph.nodes), [input_data_node, shape_0_node, gather_node,
- equal_node, unsqueeze_1_node, unsqueeze_2_node,
- concat_node, reshape_node, shape_1_node,
+
+ graph.insert_nodes(len(graph.nodes), [input_data_node, shape_0_node, gather_node,
+ equal_node, unsqueeze_1_node, unsqueeze_2_node,
+ concat_node, reshape_node, shape_1_node,
expand_node, cast_node, where_node])
graph = PaddingSequence()(graph)
self.assertEqual(3, len(graph.nodes))
@@ -351,88 +351,88 @@ def test_padding_sequence_4(self):
input_tensors = []
output_tensors = [Tensor(name='src', source_op=['input_data'],
dest_op=['reducemax', 'unsqueeze1']), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
reducemax_node = OPERATORS['OpAny']()
input_tensors = [Tensor(name='src', source_op=['input_data'], dest_op=['reducemax'])]
output_tensors = [Tensor(name='reducemax:0', source_op=['reducemax'], dest_op=['cast1'])]
- reducemax_node.construct('reducemax', 'ReduceMax', input_tensors=input_tensors,
+ reducemax_node.construct('reducemax', 'ReduceMax', input_tensors=input_tensors,
output_tensors=output_tensors)
cast1_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='reducemax:0', source_op=['reducemax'], dest_op=['cast1'])]
output_tensors = [Tensor(name='cast1:0', source_op=['cast1'], dest_op=['range'])]
- cast1_node.construct('cast1', 'Cast', input_tensors=input_tensors,
+ cast1_node.construct('cast1', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors)
range_node = OPERATORS['Range']()
input_tensors = [Tensor(name='cast1:0', source_op=['cast1'], dest_op=['range'])]
output_tensors = [Tensor(name='range:0', source_op=['range'], dest_op=['expand'])]
- range_node.construct('range', 'Range', input_tensors=input_tensors,
+ range_node.construct('range', 'Range', input_tensors=input_tensors,
output_tensors=output_tensors)
conshape_node = OPERATORS['ConstantOfShape']()
input_tensors = [Tensor(name='cons_src0:0')]
output_tensors = [Tensor(name='conshape:0', source_op=['conshape'], dest_op=['expand'])]
- conshape_node.construct('conshape', 'ConstantOfShape', input_tensors=input_tensors,
+ conshape_node.construct('conshape', 'ConstantOfShape', input_tensors=input_tensors,
output_tensors=output_tensors)
expand_node = OPERATORS['Expand']()
input_tensors = [Tensor(name='range:0', source_op=['range'], dest_op=['expand']),
Tensor(name='conshape:0', source_op=['conshape'], dest_op=['expand'])]
output_tensors = [Tensor(name='expand:0', source_op=['expand'], dest_op=['tile'])]
- expand_node.construct('expand', 'Expand', input_tensors=input_tensors,
+ expand_node.construct('expand', 'Expand', input_tensors=input_tensors,
output_tensors=output_tensors)
tile_node = OPERATORS['Tile']()
input_tensors = [Tensor(name='expand:0', source_op=['expand'], dest_op=['tile'])]
output_tensors = [Tensor(name='tile:0', source_op=['tile'], dest_op=['less'])]
- tile_node.construct('tile', 'Tile', input_tensors=input_tensors,
+ tile_node.construct('tile', 'Tile', input_tensors=input_tensors,
output_tensors=output_tensors)
unsqueeze1_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='src', source_op=['input_data'], dest_op=['unsqueeze1'])]
output_tensors = [Tensor(name='unsqueeze1:0', source_op=['unsqueeze1'], dest_op=['less'])]
- unsqueeze1_node.construct('unsqueeze1', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze1_node.construct('unsqueeze1', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
less_node = OPERATORS['Less']()
input_tensors = [Tensor(name='tile:0', source_op=['tile'], dest_op=['less']),
Tensor(name='unsqueeze1:0', source_op=['unsqueeze1'], dest_op=['less'])]
output_tensors = [Tensor(name='less:0', source_op=['less'], dest_op=['unsqueeze2'])]
- less_node.construct('less', 'Less', input_tensors=input_tensors,
+ less_node.construct('less', 'Less', input_tensors=input_tensors,
output_tensors=output_tensors)
unsqueeze2_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='less:0', source_op=['less'], dest_op=['unsqueeze2'])]
output_tensors = [Tensor(name='unsqueeze2:0', source_op=['unsqueeze2'], dest_op=['not'])]
- unsqueeze2_node.construct('unsqueeze2', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze2_node.construct('unsqueeze2', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
not_node = OPERATORS['Not']()
input_tensors = [Tensor(name='unsqueeze2:0', source_op=['unsqueeze2'], dest_op=['not'])]
output_tensors = [Tensor(name='not:0', source_op=['not'], dest_op=['unsqueeze3'])]
- not_node.construct('not', 'Not', input_tensors=input_tensors,
+ not_node.construct('not', 'Not', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
unsqueeze3_node = OPERATORS['Unsqueeze']()
input_tensors = [Tensor(name='not:0', source_op=['not'], dest_op=['unsqueeze2'])]
output_tensors = [Tensor(name='unsqueeze3:0', source_op=['unsqueeze3'], dest_op=['cast2'])]
- unsqueeze3_node.construct('unsqueeze3', 'Unsqueeze', input_tensors=input_tensors,
+ unsqueeze3_node.construct('unsqueeze3', 'Unsqueeze', input_tensors=input_tensors,
output_tensors=output_tensors)
cast2_node = OPERATORS['Cast']()
input_tensors = [Tensor(name='unsqueeze3:0', source_op=['unsqueeze3'], dest_op=['cast2'])]
output_tensors = [Tensor(name='cast2:0', source_op=['cast2'], dest_op=['where'])]
- cast2_node.construct('cast2', 'Cast', input_tensors=input_tensors,
+ cast2_node.construct('cast2', 'Cast', input_tensors=input_tensors,
output_tensors=output_tensors)
where_node = OPERATORS['Where']()
input_tensors = [Tensor(name='cast2:0', source_op=['cast2'], dest_op=['where']),
Tensor(name='where_src1'), Tensor(name='where_src2')]
output_tensors = [Tensor(name='where:0', source_op=['where'], dest_op=[])]
- where_node.construct('where', 'Where', input_tensors=input_tensors,
+ where_node.construct('where', 'Where', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, reducemax_node, cast1_node,
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_pattern_dispatch.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_pattern_dispatch.py
index 249d7b403b9..21fb339583d 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_pattern_dispatch.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_pattern_dispatch.py
@@ -43,7 +43,7 @@ def test_pattern_dispatch(self):
input_0 = np.random.uniform(low=0, high=128, size=shape).astype('int32')
input_1 = np.random.uniform(low=0, high=1, size=shape).astype('int32')
input_2 = np.random.uniform(low=0, high=1, size=shape).astype('int32')
-
+
# validate pattern tuning
fp32_model_path = "/tf_dataset2/inc-ut/nlptoolkit_ut_model/bert_mini_sst2_1x4_fp32.onnx"
if is_win():
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_position_embeddings.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_position_embeddings.py
index 7479df39703..c0d52cd7b64 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_position_embeddings.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_position_embeddings.py
@@ -32,54 +32,54 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_position_embeddings_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
slice_node = OPERATORS['Slice']()
input_tensors = [Tensor(shape=[512, 768], data=np.array(1)), Tensor(data=np.array(1))]
- output_tensors = [Tensor(name='slice:0', source_op=['slice'],
+ output_tensors = [Tensor(name='slice:0', source_op=['slice'],
dest_op=['reshape'])]
- slice_node.construct('slice', 'Slice', input_tensors=input_tensors,
+ slice_node.construct('slice', 'Slice', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
reshape_node = OPERATORS['Reshape']()
- input_tensors = [Tensor(name='slice:0', source_op=['slice'],
+ input_tensors = [Tensor(name='slice:0', source_op=['slice'],
dest_op=['reshape'])]
output_tensors = [Tensor(name='reshape:0', source_op=['reshape'],
dest_op=[])]
- reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
+ reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
less_equal_node = OPERATORS['LessEqual']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='less_equal:0', source_op=['less_equal'],
dest_op=['all'])]
- less_equal_node.construct('less_equal', 'LessEqual', input_tensors=input_tensors,
+ less_equal_node.construct('less_equal', 'LessEqual', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
all_node = OPERATORS['All']()
input_tensors = [Tensor(name='less_equal:0', source_op=['less_equal'],
dest_op=['all'])]
output_tensors = [Tensor(name='all:0', source_op=['all'],
dest_op=['assert'])]
- all_node.construct('all', 'All', input_tensors=input_tensors,
+ all_node.construct('all', 'All', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
assert_node = OPERATORS['Assert']()
input_tensors = [Tensor(name='all:0', source_op=['all'],
dest_op=['assert'])]
output_tensors = [Tensor(name='assert:0', source_op=['assert'],
dest_op=[])]
- assert_node.construct('assert', 'Assert', input_tensors=input_tensors,
+ assert_node.construct('assert', 'Assert', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, slice_node, reshape_node,
less_equal_node, all_node, assert_node])
graph = PositionEmbeddings()(graph)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_merge.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_merge.py
index 3ba348142eb..9ea0169396e 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_merge.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_merge.py
@@ -37,7 +37,7 @@ def tearDownClass(cls):
def test_qkv_merge_1(self):
model_path = "/tf_dataset2/inc-ut/nlptoolkit_ut_model/onnx_best_acc_distilbert.onnx"
content = "pattern_switch:\n 'QKVMerge': True\n 'MultiHeadAttention': False"
- pattern_config = "qkv_merge_pattern_config"
+ pattern_config = "qkv_merge_pattern_config"
with open("qkv_merge_pattern_config", "w") as file:
file.write(content)
if is_win():
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_reshape.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_reshape.py
index 8c34ba431d5..da7116a9f3b 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_reshape.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_qkv_reshape.py
@@ -31,32 +31,32 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_attention_reshape_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
matmul_node = OPERATORS['MatMulWithBias']()
input_tensors = [Tensor(), Tensor(), Tensor()]
output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['reshape'])]
- matmul_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
+ matmul_node.construct('matmul', 'MatMulWithBias', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict(
{'src1_perm': '0,1'}))
-
+
reshape_node = OPERATORS['Reshape']()
input_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['reshape'])]
output_tensors = [Tensor(name='reshape:0', source_op=['reshape'], dest_op=[])]
- reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
+ reshape_node.construct('reshape', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_shape': '0,0,12,64'}))
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, matmul_node, reshape_node])
graph = QKVReshape()(graph)
self.assertEqual(3, len(graph.nodes))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_quant_onnx_execute.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_quant_onnx_execute.py
index 3f0355846d5..1831828e21f 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_quant_onnx_execute.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_quant_onnx_execute.py
@@ -47,7 +47,7 @@ def test_onnx_qlinear_compile(self):
'ONNX QLinear model is not found, please set your own model path!')
qlinear_model = compile(qlinear_model_path)
qlinear_output_dict = qlinear_model.inference([input_0, input_1, input_2])
- qlinear_output = copy.deepcopy(list(qlinear_output_dict.values())[0])
+ qlinear_output = copy.deepcopy(list(qlinear_output_dict.values())[0])
# compile and execute qdq model
qdq_model_path = "/tf_dataset2/inc-ut/nlptoolkit_ut_model/qlinear/bert_mini_sst2_qdq.onnx"
if is_win():
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_start_end_logits.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_start_end_logits.py
index 3d83ec57a1c..460cc089be6 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_start_end_logits.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_start_end_logits.py
@@ -31,38 +31,38 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_start_end_logits(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
transpose_node = OPERATORS['Transpose']()
input_tensors = [Tensor()]
output_tensors = [Tensor(name='transpose:0', source_op=['transpose'],
dest_op=['unpack'])]
- transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
+ transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
unpack_node = OPERATORS['Unpack']()
input_tensors = [Tensor(name='transpose:0', source_op=['transpose'],
dest_op=['unpack'])]
output_tensors = [Tensor(name='unpack:0', source_op=['unpack'],
dest_op=['identity'])]
- unpack_node.construct('unpack', 'Unpack', input_tensors=input_tensors,
+ unpack_node.construct('unpack', 'Unpack', input_tensors=input_tensors,
output_tensors=output_tensors)
identity_node = OPERATORS['Identity']()
input_tensors = [Tensor(name='unpack:0', source_op=['unpack'],
dest_op=['identity'])]
output_tensors = [Tensor(name='identity:0', source_op=['identity'], dest_op=[])]
- identity_node.construct('identity', 'Identity', input_tensors=input_tensors,
+ identity_node.construct('identity', 'Identity', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, transpose_node, unpack_node,
identity_node])
graph = StartEndLogits()(graph)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_tf_utils.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_tf_utils.py
index b16f9aacdaf..fa04614461a 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_tf_utils.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_tf_utils.py
@@ -16,8 +16,8 @@
# limitations under the License.
import unittest
-from tensorflow.core.framework import node_def_pb2
-import intel_extension_for_transformers.llm.runtime.deprecated.compile.tf_utils as util
+from tensorflow.core.framework import node_def_pb2
+import intel_extension_for_transformers.llm.runtime.deprecated.compile.tf_utils as util
class TestTfUtils(unittest.TestCase):
@@ -28,7 +28,7 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_create_tf_node(self):
test_node = util.create_tf_node('Reshape', 'test_name', ['input_0'])
self.assertEqual('Reshape', test_node.op)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_token_type_embeddings.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_token_type_embeddings.py
index 0333dcc4d02..8ec6ea97771 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_token_type_embeddings.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_token_type_embeddings.py
@@ -32,76 +32,76 @@ def setUpClass(self):
@classmethod
def tearDownClass(self):
pass
-
+
def test_token_type_embeddings_1(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
reshape_1_node = OPERATORS['Reshape']()
input_tensors = [Tensor(data=np.array(1))]
output_tensors = [Tensor(name='reshape_1:0', source_op=['reshape_1'], dest_op=['one_hot'])]
- reshape_1_node.construct('reshape_1', 'Reshape', input_tensors=input_tensors,
+ reshape_1_node.construct('reshape_1', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
one_hot_node = OPERATORS['OneHot']()
- input_tensors = [Tensor(name='reshape_1:0', source_op=['reshape_1'],
+ input_tensors = [Tensor(name='reshape_1:0', source_op=['reshape_1'],
dest_op=['one_hot'])]
output_tensors = [Tensor(name='one_hot:0', source_op=['one_hot'],
dest_op=['matmul'])]
- one_hot_node.construct('one_hot', 'OneHot', input_tensors=input_tensors,
+ one_hot_node.construct('one_hot', 'OneHot', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({'test': 1}))
-
+
matmul_node = OPERATORS['MatMul']()
- input_tensors = [Tensor(name='one_hot:0', source_op=['one_hot'], dest_op=['matmul']),
+ input_tensors = [Tensor(name='one_hot:0', source_op=['one_hot'], dest_op=['matmul']),
Tensor(name='embeddings', shape=[2, 768], dest_op=['matmul'],
data=np.array(1))]
output_tensors = [Tensor(name='matmul:0', source_op=['matmul'],
dest_op=['reshape_2'])]
- matmul_node.construct('matmul', 'MatMul', input_tensors=input_tensors,
+ matmul_node.construct('matmul', 'MatMul', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict(
{'transpose_a': False, 'transpose_b': False}))
-
+
shape_node = OPERATORS['Shape']()
input_tensors = [Tensor(np.array(1))]
- output_tensors = [Tensor(name='shape:0', source_op=['shape'],
+ output_tensors = [Tensor(name='shape:0', source_op=['shape'],
dest_op=['strided_slice'])]
- shape_node.construct('shape', 'Shape', input_tensors=input_tensors,
+ shape_node.construct('shape', 'Shape', input_tensors=input_tensors,
output_tensors=output_tensors)
strided_slice_node = OPERATORS['StridedSlice']()
- input_tensors = [Tensor(name='shape:0', source_op=['shape'],
+ input_tensors = [Tensor(name='shape:0', source_op=['shape'],
dest_op=['strided_slice'])]
- output_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
+ output_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['pack'])]
- strided_slice_node.construct('strided_slice', 'StridedSlice', input_tensors=input_tensors,
+ strided_slice_node.construct('strided_slice', 'StridedSlice', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
pack_node = OPERATORS['Pack']()
- input_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
+ input_tensors = [Tensor(name='strided_slice:0', source_op=['strided_slice'],
dest_op=['pack'])]
output_tensors = [Tensor(name='pack:0', source_op=['pack'], dest_op=['reshape_2'])]
- pack_node.construct('pack', 'Pack', input_tensors=input_tensors,
+ pack_node.construct('pack', 'Pack', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
reshape_2_node = OPERATORS['Reshape']()
input_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['reshape_2']),
Tensor(name='pack:0', source_op=['pack'], dest_op=['reshape_2'])]
output_tensors = [Tensor(name='reshape_2:0', source_op=['reshape_2'])]
- reshape_2_node.construct('reshape_2', 'Reshape', input_tensors=input_tensors,
+ reshape_2_node.construct('reshape_2', 'Reshape', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
graph.insert_nodes(len(graph.nodes), [input_data_node, reshape_1_node, one_hot_node,
matmul_node, shape_node, strided_slice_node,
pack_node, reshape_2_node])
-
+
graph = TokenTypeEmbeddings()(graph)
self.assertEqual(6, len(graph.nodes))
self.assertEqual(-1, graph.nodes[1].attr['dst_shape'])
@@ -109,38 +109,38 @@ def test_token_type_embeddings_1(self):
self.assertEqual('1,0', graph.nodes[3].attr['src1_perm'])
self.assertEqual('-1,-1,768', graph.nodes[4].attr['dst_shape'])
self.assertEqual('1,2', graph.nodes[5].attr['mul'])
-
-
+
+
def test_token_type_embeddings_2(self):
graph = Graph()
graph.framework_modeling_config['framework'] = 'onnxruntime'
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(name='segment_ids', dest_op=['gather']), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
gather_node = OPERATORS['Gather']()
- input_tensors = [Tensor(shape=[2, 768], data=np.array(1)), Tensor(name='segment_ids',
+ input_tensors = [Tensor(shape=[2, 768], data=np.array(1)), Tensor(name='segment_ids',
source_op=['input_data'], dest_op=['gather'])]
output_tensors = [Tensor(name='gather:0', source_op=['gather'], dest_op=['add'])]
- gather_node.construct('gather', 'Gather', input_tensors=input_tensors,
+ gather_node.construct('gather', 'Gather', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'axis': 0, 'batch_dims': 0}))
-
+
add_node = OPERATORS['Add']()
input_tensors = [Tensor(name='gather:0', source_op=['gather'], dest_op=['add']), Tensor(
data=np.array(1))]
output_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['layer_norm'])]
- add_node.construct('add', 'Add', input_tensors=input_tensors,
+ add_node.construct('add', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
-
+
ln_node = OPERATORS['LayerNorm']()
- input_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['layer_norm']),
- Tensor(shape=[768], data=np.array(1)),
+ input_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=['layer_norm']),
+ Tensor(shape=[768], data=np.array(1)),
Tensor(shape=[768], data=np.array(1))]
output_tensors = [Tensor(name='layer_norm:0', source_op=['layer_norm'], dest_op=[])]
- ln_node.construct('layer_norm', 'LayerNorm', input_tensors=input_tensors,
+ ln_node.construct('layer_norm', 'LayerNorm', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, gather_node, add_node, ln_node])
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_transpose_batch_matmul.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_transpose_batch_matmul.py
index f74bcba6860..b0734286892 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_transpose_batch_matmul.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_transpose_batch_matmul.py
@@ -39,14 +39,14 @@ def test_transpose_batch_matmul_1(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
transpose_1_node = OPERATORS['Transpose']()
input_tensors = [Tensor(data=np.array(1))]
output_tensors = [Tensor(name='transpose_1:0', source_op=['transpose_1'],
dest_op=['fused_matmul'])]
- transpose_1_node.construct('transpose_1', 'Transpose', input_tensors=input_tensors,
+ transpose_1_node.construct('transpose_1', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_perm': '0,2,1,3'}))
@@ -54,17 +54,17 @@ def test_transpose_batch_matmul_1(self):
input_tensors = [Tensor(data=np.array(1))]
output_tensors = [Tensor(name='transpose_2:0', source_op=['transpose_2'],
dest_op=['fused_matmul'])]
- transpose_2_node.construct('transpose_2', 'Transpose', input_tensors=input_tensors,
+ transpose_2_node.construct('transpose_2', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_perm': '0,2,3,1'}))
fused_matmul_node = OPERATORS['FusedMatMul']()
input_tensors = [Tensor(name='transpose_1:0', source_op=['transpose_1'],
- dest_op=['fused_matmul']), Tensor(name='transpose_2:0',
+ dest_op=['fused_matmul']), Tensor(name='transpose_2:0',
source_op=['transpose_2'], dest_op=['fused_matmul'])]
output_tensors = [Tensor(name='fused_matmul:0', source_op=['fused_matmul'],
dest_op=['add'])]
- fused_matmul_node.construct('fused_matmul', 'FusedMatMul', input_tensors=input_tensors,
+ fused_matmul_node.construct('fused_matmul', 'FusedMatMul', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'transpose_a': False, 'transpose_b': False, 'alpha': 0.125}))
@@ -72,7 +72,7 @@ def test_transpose_batch_matmul_1(self):
input_tensors = [Tensor(name='fused_matmul:0', source_op=['fused_matmul'],
dest_op=['add']), Tensor(data=np.array(1))]
output_tensors = [Tensor(name='add:0', source_op=['add'], dest_op=[])]
- add_node.construct('add', 'Add', input_tensors=input_tensors,
+ add_node.construct('add', 'Add', input_tensors=input_tensors,
output_tensors=output_tensors)
graph.insert_nodes(len(graph.nodes), [input_data_node, transpose_1_node, transpose_2_node,
@@ -90,14 +90,14 @@ def test_transpose_batch_matmul_2(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
transpose_1_node = OPERATORS['Transpose']()
input_tensors = [Tensor(data=np.array(1))]
output_tensors = [Tensor(name='transpose_1:0', source_op=['transpose_1'],
dest_op=['matmul'])]
- transpose_1_node.construct('transpose_1', 'Transpose', input_tensors=input_tensors,
+ transpose_1_node.construct('transpose_1', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_perm': '0,2,1,3'}))
@@ -105,14 +105,14 @@ def test_transpose_batch_matmul_2(self):
input_tensors = [Tensor(name='transpose_1:0', source_op=['transpose_1'],
dest_op=['matmul']), Tensor(name='m_src1:0')]
output_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['transpose_2'])]
- matmul_node.construct('matmul', 'BatchMatMul', input_tensors=input_tensors,
+ matmul_node.construct('matmul', 'BatchMatMul', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'transpose_a': False, 'transpose_b': False}))
transpose_2_node = OPERATORS['Transpose']()
input_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['transpose_2'])]
output_tensors = [Tensor(name='transpose_2:0', source_op=['transpose_2'])]
- transpose_2_node.construct('transpose_2', 'Transpose', input_tensors=input_tensors,
+ transpose_2_node.construct('transpose_2', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_perm': '0,2,3,1'}))
@@ -129,20 +129,20 @@ def test_transpose_batch_matmul_3(self):
input_data_node = OPERATORS['Input']()
input_tensors = []
output_tensors = [Tensor(), Tensor(), Tensor()]
- input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
+ input_data_node.construct('input_data', 'Input', input_tensors=input_tensors,
output_tensors=output_tensors)
matmul_node = OPERATORS['BatchMatMul']()
input_tensors = [Tensor(name='m_src0:0'), Tensor(name='m_src1:0')]
output_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['transpose'])]
- matmul_node.construct('matmul', 'BatchMatMul', input_tensors=input_tensors,
+ matmul_node.construct('matmul', 'BatchMatMul', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'transpose_a': False, 'transpose_b': False}))
transpose_node = OPERATORS['Transpose']()
input_tensors = [Tensor(name='matmul:0', source_op=['matmul'], dest_op=['transpose'])]
output_tensors = [Tensor(name='transpose:0', source_op=['transpose'])]
- transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
+ transpose_node.construct('transpose', 'Transpose', input_tensors=input_tensors,
output_tensors=output_tensors, attr=OrderedDict({
'dst_perm': '0,2,3,1'}))
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_unet_qat.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_unet_qat.py
index 274f17192cd..8cb494a3aee 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_unet_qat.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/test_unet_qat.py
@@ -119,7 +119,7 @@ def test_unet(self):
self.assertTrue(os.path.exists(model_dir), 'model is not found, please set your own model path!')
graph = compile(model_dir, config=qat_unet_pattern_config)
-
+
input_0_path = root_dir + 'sample.pt'
inputs_0 = torch.load(input_0_path)
inputs_1 = torch.tensor([301], dtype=torch.float32)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_add_zeros.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_add_zeros.py
index d1ef664c9f5..bb00d4e73cd 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_add_zeros.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_add_zeros.py
@@ -50,10 +50,10 @@ def test_1(self):
n = Net()
example_in = torch.rand(3, 256)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_baddbmm.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_baddbmm.py
index e5efc6ff408..ffa5a793223 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_baddbmm.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_baddbmm.py
@@ -51,10 +51,10 @@ def test_1(self):
batch1 = torch.randn(10, 3, 4)
batch2 = torch.randn(10, 4, 5)
traced_model = torch.jit.trace(n, (M, batch1, batch2))
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(M, batch1, batch2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
@@ -71,10 +71,10 @@ def test_2(self):
batch1 = torch.randn(10, 3, 4)
batch2 = torch.randn(10, 4, 5)
traced_model = torch.jit.trace(n, (M, batch1, batch2))
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(M, batch1, batch2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_bert_mini_fp32.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_bert_mini_fp32.py
index d2bb2f4406d..a4115c5cf93 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_bert_mini_fp32.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_bert_mini_fp32.py
@@ -53,7 +53,7 @@ def test_1(self):
example_in = torch.rand(8, 128)
# TODO: enable check accuracy
ref_out = traced_model(ids, tok, att, ids)[0].detach().numpy()
-
+
graph = compile(pt_file)
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_binaryop.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_binaryop.py
index 5fabb9c3da6..0c85756c5c7 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_binaryop.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_binaryop.py
@@ -65,7 +65,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, (example_in, example_in2))
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
@@ -83,7 +83,7 @@ def test_2(self):
traced_model = torch.jit.trace(n, (example_in, example_in2))
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
@@ -99,10 +99,10 @@ def test_2(self):
# example_in = torch.rand(3, 256)
# example_in2 = torch.rand(256)
# traced_model = torch.jit.trace(n, (example_in, example_in2))
- #
+ #
# torch.jit.save(traced_model, '{}.pt'.format(file_name))
# ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
# graph = compile('{}.pt'.format(file_name))
# graph.save(file_name)
# newgraph = Graph()
@@ -118,10 +118,10 @@ def test_2(self):
# example_in = torch.rand(3, 256)
# example_in2 = torch.rand(256)
# traced_model = torch.jit.trace(n, (example_in, example_in2))
- #
+ #
# torch.jit.save(traced_model, '{}.pt'.format(file_name))
# ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
# graph = compile('{}.pt'.format(file_name))
# graph.save(file_name)
# newgraph = Graph()
@@ -137,10 +137,10 @@ def test_2(self):
# example_in = torch.rand(3, 256)
# example_in2 = torch.rand(256)
# traced_model = torch.jit.trace(n, (example_in, example_in2))
- #
+ #
# torch.jit.save(traced_model, '{}.pt'.format(file_name))
# ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
# graph = compile('{}.pt'.format(file_name))
# graph.save(file_name)
# newgraph = Graph()
@@ -156,10 +156,10 @@ def test_2(self):
# example_in = torch.rand(3, 256)
# example_in2 = torch.rand(256)
# traced_model = torch.jit.trace(n, (example_in, example_in2))
- #
+ #
# torch.jit.save(traced_model, '{}.pt'.format(file_name))
# ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
# graph = compile('{}.pt'.format(file_name))
# graph.save(file_name)
# newgraph = Graph()
@@ -175,10 +175,10 @@ def test_2(self):
# example_in = torch.rand(3, 256)
# example_in2 = torch.rand(256)
# traced_model = torch.jit.trace(n, (example_in, example_in2))
- #
+ #
# torch.jit.save(traced_model, '{}.pt'.format(file_name))
# ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
# graph = compile('{}.pt'.format(file_name))
# graph.save(file_name)
# newgraph = Graph()
@@ -194,10 +194,10 @@ def test_2(self):
# example_in = torch.rand(3, 256)
# example_in2 = torch.rand(256)
# traced_model = torch.jit.trace(n, (example_in, example_in2))
- #
+ #
# torch.jit.save(traced_model, '{}.pt'.format(file_name))
# ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
# graph = compile('{}.pt'.format(file_name))
# graph.save(file_name)
# newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_concat.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_concat.py
index 4263abda43c..2f756857d2b 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_concat.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_concat.py
@@ -51,7 +51,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, (example_in, example_in2))
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv.py
index c154711d103..6918d9d29dc 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv.py
@@ -57,7 +57,7 @@ def test_1(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
np.testing.assert_almost_equal(ref_out, [*out.values()][0], decimal=5)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv_int8.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv_int8.py
index 98dafc56645..71406e9c94e 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv_int8.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_conv_int8.py
@@ -59,7 +59,7 @@ def test_1(self):
from torch.ao.quantization import MinMaxObserver, PerChannelMinMaxObserver, QConfig
qconfig = QConfig(activation=MinMaxObserver.with_args(qscheme=torch.per_tensor_affine, dtype=torch.quint8),
weight=MinMaxObserver.with_args(dtype=torch.qint8, qscheme=torch.per_tensor_symmetric))
-
+
n = Net().eval()
n.apply(weight_init)
example_in = torch.rand(3, 16, 13, 13)
@@ -76,7 +76,7 @@ def test_1(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
np.testing.assert_almost_equal(ref_out, [*out.values()][0], decimal=5)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
@@ -85,7 +85,7 @@ def test_2(self):
from torch.ao.quantization import MinMaxObserver, PerChannelMinMaxObserver, QConfig
qconfig = QConfig(activation=MinMaxObserver.with_args(qscheme=torch.per_tensor_affine, dtype=torch.quint8),
weight=PerChannelMinMaxObserver.with_args(dtype=torch.qint8, qscheme=torch.per_channel_symmetric))
-
+
n = Net().eval()
n.apply(weight_init)
example_in = torch.rand(3, 16, 13, 13)
@@ -102,7 +102,7 @@ def test_2(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
np.testing.assert_almost_equal(ref_out, [*out.values()][0], decimal=5)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_dolly_pattern.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_dolly_pattern.py
index 11ab75c6793..676200c87e2 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_dolly_pattern.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_dolly_pattern.py
@@ -1089,4 +1089,4 @@ def test_1(self):
dollygraph = p_fusion(dollygraph)
if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_embedding.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_embedding.py
index 2e07d81c900..31a2f9eaa10 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_embedding.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_embedding.py
@@ -53,7 +53,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, example_in)
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).squeeze(0).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gather.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gather.py
index 8db816f58d8..3675c250b95 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gather.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gather.py
@@ -49,11 +49,11 @@ def test_1(self):
n = Net()
example_in = torch.randint(0, 22, (1, 10))
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
print(ref_out.shape)
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gelu.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gelu.py
index eba3026e5ae..a7530295ef5 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gelu.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gelu.py
@@ -56,10 +56,10 @@ def test_1(self):
n = Net()
example_in = torch.rand(3, 256)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
@@ -74,17 +74,17 @@ def test_2(self):
n = Net2()
example_in = torch.rand(3, 256)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
- np.testing.assert_almost_equal(ref_out, [*out.values()][0], decimal=5)
+ np.testing.assert_almost_equal(ref_out, [*out.values()][0], decimal=5)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gpt_int8.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gpt_int8.py
index b36008632f7..989b4f39014 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gpt_int8.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_gpt_int8.py
@@ -153,4 +153,4 @@ def test_1(self):
os.remove('int8_pattern.conf')
if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_layernorm.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_layernorm.py
index 2efc3d5fbc8..168d81e89b5 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_layernorm.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_layernorm.py
@@ -49,11 +49,11 @@ def test_1(self):
n = Net()
example_in = torch.randn(20, 5, 10)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
# torch.onnx.export(n, example_in, '{}.onnx'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear.py
index 63a3635745d..fd3e122a20c 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear.py
@@ -57,7 +57,7 @@ def test_1(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
np.testing.assert_almost_equal(ref_out, [*out.values()][0], decimal=5)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear_int8.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear_int8.py
index b501a49715f..eb2970ccab1 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear_int8.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_linear_int8.py
@@ -88,7 +88,7 @@ def test_per_tensor(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
self.assertTrue(cmpData(ref_out, [*out.values()][0]) < 0.01)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
@@ -114,7 +114,7 @@ def test_per_channel(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
self.assertTrue(cmpData(ref_out, [*out.values()][0]) < 0.01)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
@@ -140,7 +140,7 @@ def test_per_tensor_wo_bias(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
self.assertTrue(cmpData(ref_out, [*out.values()][0]) < 0.01)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
@@ -166,7 +166,7 @@ def test_per_channel_wo_bias(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
self.assertTrue(cmpData(ref_out, [*out.values()][0]) < 0.01)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_llama_pattern.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_llama_pattern.py
index d3d22b73622..06398c328bb 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_llama_pattern.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_llama_pattern.py
@@ -293,4375 +293,4375 @@ def test_1(self):
'479':
dtype: fp32
shape: [32000, 5120]
-
+
'483':
dtype: s64
shape: [1]
-
+
aten::pow_2539_other:
dtype: fp32
shape: [1]
-
+
'485':
dtype: fp32
shape: [1]
-
+
'486':
dtype: fp32
shape: [5120]
-
+
'487':
dtype: fp32
shape: [5120]
-
+
'488':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3436_bias:
dtype: fp32
shape: [5120]
-
+
'489':
dtype: fp32
shape: [5120]
-
+
'490':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3437_bias:
dtype: fp32
shape: [5120]
-
+
'491':
dtype: fp32
shape: [5120]
-
+
'492':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3438_bias:
dtype: fp32
shape: [5120]
-
+
'493':
dtype: fp32
shape: [1, 1, 2048, 128]
-
+
'494':
dtype: fp32
shape: [1, 1, 2048, 128]
-
+
'495':
dtype: s64
shape: [1]
-
+
aten::neg_4508_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4510_mul_val:
dtype: fp32
shape: [1]
-
+
'496':
dtype: fp32
shape: [1]
-
+
'497':
dtype: fp32
shape: [1]
-
+
'498':
dtype: fp32
shape: [5120]
-
+
'499':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3439_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2550_other:
dtype: fp32
shape: [1]
-
+
'500':
dtype: fp32
shape: [5120]
-
+
'501':
dtype: fp32
shape: [5120]
-
+
'502':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3440_bias:
dtype: fp32
shape: [13824]
-
+
'503':
dtype: fp32
shape: [5120]
-
+
'504':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3441_bias:
dtype: fp32
shape: [13824]
-
+
'505':
dtype: fp32
shape: [13824]
-
+
'506':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3442_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2551_other:
dtype: fp32
shape: [1]
-
+
'507':
dtype: fp32
shape: [5120]
-
+
'508':
dtype: fp32
shape: [5120]
-
+
'509':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3443_bias:
dtype: fp32
shape: [5120]
-
+
'510':
dtype: fp32
shape: [5120]
-
+
'511':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3444_bias:
dtype: fp32
shape: [5120]
-
+
'512':
dtype: fp32
shape: [5120]
-
+
'513':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3445_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4540_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4542_mul_val:
dtype: fp32
shape: [1]
-
+
'514':
dtype: fp32
shape: [5120]
-
+
'515':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3446_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2563_other:
dtype: fp32
shape: [1]
-
+
'516':
dtype: fp32
shape: [5120]
-
+
'517':
dtype: fp32
shape: [5120]
-
+
'518':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3447_bias:
dtype: fp32
shape: [13824]
-
+
'519':
dtype: fp32
shape: [5120]
-
+
'520':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3448_bias:
dtype: fp32
shape: [13824]
-
+
'521':
dtype: fp32
shape: [13824]
-
+
'522':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3449_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2564_other:
dtype: fp32
shape: [1]
-
+
'523':
dtype: fp32
shape: [5120]
-
+
'524':
dtype: fp32
shape: [5120]
-
+
'525':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3450_bias:
dtype: fp32
shape: [5120]
-
+
'526':
dtype: fp32
shape: [5120]
-
+
'527':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3451_bias:
dtype: fp32
shape: [5120]
-
+
'528':
dtype: fp32
shape: [5120]
-
+
'529':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3452_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4572_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4574_mul_val:
dtype: fp32
shape: [1]
-
+
'530':
dtype: fp32
shape: [5120]
-
+
'531':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3453_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2576_other:
dtype: fp32
shape: [1]
-
+
'532':
dtype: fp32
shape: [5120]
-
+
'533':
dtype: fp32
shape: [5120]
-
+
'534':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3454_bias:
dtype: fp32
shape: [13824]
-
+
'535':
dtype: fp32
shape: [5120]
-
+
'536':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3455_bias:
dtype: fp32
shape: [13824]
-
+
'537':
dtype: fp32
shape: [13824]
-
+
'538':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3456_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2577_other:
dtype: fp32
shape: [1]
-
+
'539':
dtype: fp32
shape: [5120]
-
+
'540':
dtype: fp32
shape: [5120]
-
+
'541':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3457_bias:
dtype: fp32
shape: [5120]
-
+
'542':
dtype: fp32
shape: [5120]
-
+
'543':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3458_bias:
dtype: fp32
shape: [5120]
-
+
'544':
dtype: fp32
shape: [5120]
-
+
'545':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3459_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4604_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4606_mul_val:
dtype: fp32
shape: [1]
-
+
'546':
dtype: fp32
shape: [5120]
-
+
'547':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3460_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2589_other:
dtype: fp32
shape: [1]
-
+
'548':
dtype: fp32
shape: [5120]
-
+
'549':
dtype: fp32
shape: [5120]
-
+
'550':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3461_bias:
dtype: fp32
shape: [13824]
-
+
'551':
dtype: fp32
shape: [5120]
-
+
'552':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3462_bias:
dtype: fp32
shape: [13824]
-
+
'553':
dtype: fp32
shape: [13824]
-
+
'554':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3463_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2590_other:
dtype: fp32
shape: [1]
-
+
'555':
dtype: fp32
shape: [5120]
-
+
'556':
dtype: fp32
shape: [5120]
-
+
'557':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3464_bias:
dtype: fp32
shape: [5120]
-
+
'558':
dtype: fp32
shape: [5120]
-
+
'559':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3465_bias:
dtype: fp32
shape: [5120]
-
+
'560':
dtype: fp32
shape: [5120]
-
+
'561':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3466_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4636_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4638_mul_val:
dtype: fp32
shape: [1]
-
+
'562':
dtype: fp32
shape: [5120]
-
+
'563':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3467_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2602_other:
dtype: fp32
shape: [1]
-
+
'564':
dtype: fp32
shape: [5120]
-
+
'565':
dtype: fp32
shape: [5120]
-
+
'566':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3468_bias:
dtype: fp32
shape: [13824]
-
+
'567':
dtype: fp32
shape: [5120]
-
+
'568':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3469_bias:
dtype: fp32
shape: [13824]
-
+
'569':
dtype: fp32
shape: [13824]
-
+
'570':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3470_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2603_other:
dtype: fp32
shape: [1]
-
+
'571':
dtype: fp32
shape: [5120]
-
+
'572':
dtype: fp32
shape: [5120]
-
+
'573':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3471_bias:
dtype: fp32
shape: [5120]
-
+
'574':
dtype: fp32
shape: [5120]
-
+
'575':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3472_bias:
dtype: fp32
shape: [5120]
-
+
'576':
dtype: fp32
shape: [5120]
-
+
'577':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3473_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4668_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4670_mul_val:
dtype: fp32
shape: [1]
-
+
'578':
dtype: fp32
shape: [5120]
-
+
'579':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3474_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2615_other:
dtype: fp32
shape: [1]
-
+
'580':
dtype: fp32
shape: [5120]
-
+
'581':
dtype: fp32
shape: [5120]
-
+
'582':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3475_bias:
dtype: fp32
shape: [13824]
-
+
'583':
dtype: fp32
shape: [5120]
-
+
'584':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3476_bias:
dtype: fp32
shape: [13824]
-
+
'585':
dtype: fp32
shape: [13824]
-
+
'586':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3477_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2616_other:
dtype: fp32
shape: [1]
-
+
'587':
dtype: fp32
shape: [5120]
-
+
'588':
dtype: fp32
shape: [5120]
-
+
'589':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3478_bias:
dtype: fp32
shape: [5120]
-
+
'590':
dtype: fp32
shape: [5120]
-
+
'591':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3479_bias:
dtype: fp32
shape: [5120]
-
+
'592':
dtype: fp32
shape: [5120]
-
+
'593':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3480_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4700_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4702_mul_val:
dtype: fp32
shape: [1]
-
+
'594':
dtype: fp32
shape: [5120]
-
+
'595':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3481_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2628_other:
dtype: fp32
shape: [1]
-
+
'596':
dtype: fp32
shape: [5120]
-
+
'597':
dtype: fp32
shape: [5120]
-
+
'598':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3482_bias:
dtype: fp32
shape: [13824]
-
+
'599':
dtype: fp32
shape: [5120]
-
+
'600':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3483_bias:
dtype: fp32
shape: [13824]
-
+
'601':
dtype: fp32
shape: [13824]
-
+
'602':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3484_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2629_other:
dtype: fp32
shape: [1]
-
+
'603':
dtype: fp32
shape: [5120]
-
+
'604':
dtype: fp32
shape: [5120]
-
+
'605':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3485_bias:
dtype: fp32
shape: [5120]
-
+
'606':
dtype: fp32
shape: [5120]
-
+
'607':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3486_bias:
dtype: fp32
shape: [5120]
-
+
'608':
dtype: fp32
shape: [5120]
-
+
'609':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3487_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4732_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4734_mul_val:
dtype: fp32
shape: [1]
-
+
'610':
dtype: fp32
shape: [5120]
-
+
'611':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3488_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2641_other:
dtype: fp32
shape: [1]
-
+
'612':
dtype: fp32
shape: [5120]
-
+
'613':
dtype: fp32
shape: [5120]
-
+
'614':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3489_bias:
dtype: fp32
shape: [13824]
-
+
'615':
dtype: fp32
shape: [5120]
-
+
'616':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3490_bias:
dtype: fp32
shape: [13824]
-
+
'617':
dtype: fp32
shape: [13824]
-
+
'618':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3491_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2642_other:
dtype: fp32
shape: [1]
-
+
'619':
dtype: fp32
shape: [5120]
-
+
'620':
dtype: fp32
shape: [5120]
-
+
'621':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3492_bias:
dtype: fp32
shape: [5120]
-
+
'622':
dtype: fp32
shape: [5120]
-
+
'623':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3493_bias:
dtype: fp32
shape: [5120]
-
+
'624':
dtype: fp32
shape: [5120]
-
+
'625':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3494_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4764_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4766_mul_val:
dtype: fp32
shape: [1]
-
+
'626':
dtype: fp32
shape: [5120]
-
+
'627':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3495_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2654_other:
dtype: fp32
shape: [1]
-
+
'628':
dtype: fp32
shape: [5120]
-
+
'629':
dtype: fp32
shape: [5120]
-
+
'630':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3496_bias:
dtype: fp32
shape: [13824]
-
+
'631':
dtype: fp32
shape: [5120]
-
+
'632':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3497_bias:
dtype: fp32
shape: [13824]
-
+
'633':
dtype: fp32
shape: [13824]
-
+
'634':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3498_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2655_other:
dtype: fp32
shape: [1]
-
+
'635':
dtype: fp32
shape: [5120]
-
+
'636':
dtype: fp32
shape: [5120]
-
+
'637':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3499_bias:
dtype: fp32
shape: [5120]
-
+
'638':
dtype: fp32
shape: [5120]
-
+
'639':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3500_bias:
dtype: fp32
shape: [5120]
-
+
'640':
dtype: fp32
shape: [5120]
-
+
'641':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3501_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4796_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4798_mul_val:
dtype: fp32
shape: [1]
-
+
'642':
dtype: fp32
shape: [5120]
-
+
'643':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3502_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2667_other:
dtype: fp32
shape: [1]
-
+
'644':
dtype: fp32
shape: [5120]
-
+
'645':
dtype: fp32
shape: [5120]
-
+
'646':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3503_bias:
dtype: fp32
shape: [13824]
-
+
'647':
dtype: fp32
shape: [5120]
-
+
'648':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3504_bias:
dtype: fp32
shape: [13824]
-
+
'649':
dtype: fp32
shape: [13824]
-
+
'650':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3505_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2668_other:
dtype: fp32
shape: [1]
-
+
'651':
dtype: fp32
shape: [5120]
-
+
'652':
dtype: fp32
shape: [5120]
-
+
'653':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3506_bias:
dtype: fp32
shape: [5120]
-
+
'654':
dtype: fp32
shape: [5120]
-
+
'655':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3507_bias:
dtype: fp32
shape: [5120]
-
+
'656':
dtype: fp32
shape: [5120]
-
+
'657':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3508_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4828_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4830_mul_val:
dtype: fp32
shape: [1]
-
+
'658':
dtype: fp32
shape: [5120]
-
+
'659':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3509_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2680_other:
dtype: fp32
shape: [1]
-
+
'660':
dtype: fp32
shape: [5120]
-
+
'661':
dtype: fp32
shape: [5120]
-
+
'662':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3510_bias:
dtype: fp32
shape: [13824]
-
+
'663':
dtype: fp32
shape: [5120]
-
+
'664':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3511_bias:
dtype: fp32
shape: [13824]
-
+
'665':
dtype: fp32
shape: [13824]
-
+
'666':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3512_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2681_other:
dtype: fp32
shape: [1]
-
+
'667':
dtype: fp32
shape: [5120]
-
+
'668':
dtype: fp32
shape: [5120]
-
+
'669':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3513_bias:
dtype: fp32
shape: [5120]
-
+
'670':
dtype: fp32
shape: [5120]
-
+
'671':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3514_bias:
dtype: fp32
shape: [5120]
-
+
'672':
dtype: fp32
shape: [5120]
-
+
'673':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3515_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4860_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4862_mul_val:
dtype: fp32
shape: [1]
-
+
'674':
dtype: fp32
shape: [5120]
-
+
'675':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3516_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2693_other:
dtype: fp32
shape: [1]
-
+
'676':
dtype: fp32
shape: [5120]
-
+
'677':
dtype: fp32
shape: [5120]
-
+
'678':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3517_bias:
dtype: fp32
shape: [13824]
-
+
'679':
dtype: fp32
shape: [5120]
-
+
'680':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3518_bias:
dtype: fp32
shape: [13824]
-
+
'681':
dtype: fp32
shape: [13824]
-
+
'682':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3519_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2694_other:
dtype: fp32
shape: [1]
-
+
'683':
dtype: fp32
shape: [5120]
-
+
'684':
dtype: fp32
shape: [5120]
-
+
'685':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3520_bias:
dtype: fp32
shape: [5120]
-
+
'686':
dtype: fp32
shape: [5120]
-
+
'687':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3521_bias:
dtype: fp32
shape: [5120]
-
+
'688':
dtype: fp32
shape: [5120]
-
+
'689':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3522_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4892_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4894_mul_val:
dtype: fp32
shape: [1]
-
+
'690':
dtype: fp32
shape: [5120]
-
+
'691':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3523_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2706_other:
dtype: fp32
shape: [1]
-
+
'692':
dtype: fp32
shape: [5120]
-
+
'693':
dtype: fp32
shape: [5120]
-
+
'694':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3524_bias:
dtype: fp32
shape: [13824]
-
+
'695':
dtype: fp32
shape: [5120]
-
+
'696':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3525_bias:
dtype: fp32
shape: [13824]
-
+
'697':
dtype: fp32
shape: [13824]
-
+
'698':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3526_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2707_other:
dtype: fp32
shape: [1]
-
+
'699':
dtype: fp32
shape: [5120]
-
+
'700':
dtype: fp32
shape: [5120]
-
+
'701':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3527_bias:
dtype: fp32
shape: [5120]
-
+
'702':
dtype: fp32
shape: [5120]
-
+
'703':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3528_bias:
dtype: fp32
shape: [5120]
-
+
'704':
dtype: fp32
shape: [5120]
-
+
'705':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3529_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4924_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4926_mul_val:
dtype: fp32
shape: [1]
-
+
'706':
dtype: fp32
shape: [5120]
-
+
'707':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3530_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2719_other:
dtype: fp32
shape: [1]
-
+
'708':
dtype: fp32
shape: [5120]
-
+
'709':
dtype: fp32
shape: [5120]
-
+
'710':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3531_bias:
dtype: fp32
shape: [13824]
-
+
'711':
dtype: fp32
shape: [5120]
-
+
'712':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3532_bias:
dtype: fp32
shape: [13824]
-
+
'713':
dtype: fp32
shape: [13824]
-
+
'714':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3533_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2720_other:
dtype: fp32
shape: [1]
-
+
'715':
dtype: fp32
shape: [5120]
-
+
'716':
dtype: fp32
shape: [5120]
-
+
'717':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3534_bias:
dtype: fp32
shape: [5120]
-
+
'718':
dtype: fp32
shape: [5120]
-
+
'719':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3535_bias:
dtype: fp32
shape: [5120]
-
+
'720':
dtype: fp32
shape: [5120]
-
+
'721':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3536_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4956_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4958_mul_val:
dtype: fp32
shape: [1]
-
+
'722':
dtype: fp32
shape: [5120]
-
+
'723':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3537_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2732_other:
dtype: fp32
shape: [1]
-
+
'724':
dtype: fp32
shape: [5120]
-
+
'725':
dtype: fp32
shape: [5120]
-
+
'726':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3538_bias:
dtype: fp32
shape: [13824]
-
+
'727':
dtype: fp32
shape: [5120]
-
+
'728':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3539_bias:
dtype: fp32
shape: [13824]
-
+
'729':
dtype: fp32
shape: [13824]
-
+
'730':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3540_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2733_other:
dtype: fp32
shape: [1]
-
+
'731':
dtype: fp32
shape: [5120]
-
+
'732':
dtype: fp32
shape: [5120]
-
+
'733':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3541_bias:
dtype: fp32
shape: [5120]
-
+
'734':
dtype: fp32
shape: [5120]
-
+
'735':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3542_bias:
dtype: fp32
shape: [5120]
-
+
'736':
dtype: fp32
shape: [5120]
-
+
'737':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3543_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_4988_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_4990_mul_val:
dtype: fp32
shape: [1]
-
+
'738':
dtype: fp32
shape: [5120]
-
+
'739':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3544_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2745_other:
dtype: fp32
shape: [1]
-
+
'740':
dtype: fp32
shape: [5120]
-
+
'741':
dtype: fp32
shape: [5120]
-
+
'742':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3545_bias:
dtype: fp32
shape: [13824]
-
+
'743':
dtype: fp32
shape: [5120]
-
+
'744':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3546_bias:
dtype: fp32
shape: [13824]
-
+
'745':
dtype: fp32
shape: [13824]
-
+
'746':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3547_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2746_other:
dtype: fp32
shape: [1]
-
+
'747':
dtype: fp32
shape: [5120]
-
+
'748':
dtype: fp32
shape: [5120]
-
+
'749':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3548_bias:
dtype: fp32
shape: [5120]
-
+
'750':
dtype: fp32
shape: [5120]
-
+
'751':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3549_bias:
dtype: fp32
shape: [5120]
-
+
'752':
dtype: fp32
shape: [5120]
-
+
'753':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3550_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5020_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5022_mul_val:
dtype: fp32
shape: [1]
-
+
'754':
dtype: fp32
shape: [5120]
-
+
'755':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3551_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2758_other:
dtype: fp32
shape: [1]
-
+
'756':
dtype: fp32
shape: [5120]
-
+
'757':
dtype: fp32
shape: [5120]
-
+
'758':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3552_bias:
dtype: fp32
shape: [13824]
-
+
'759':
dtype: fp32
shape: [5120]
-
+
'760':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3553_bias:
dtype: fp32
shape: [13824]
-
+
'761':
dtype: fp32
shape: [13824]
-
+
'762':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3554_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2759_other:
dtype: fp32
shape: [1]
-
+
'763':
dtype: fp32
shape: [5120]
-
+
'764':
dtype: fp32
shape: [5120]
-
+
'765':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3555_bias:
dtype: fp32
shape: [5120]
-
+
'766':
dtype: fp32
shape: [5120]
-
+
'767':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3556_bias:
dtype: fp32
shape: [5120]
-
+
'768':
dtype: fp32
shape: [5120]
-
+
'769':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3557_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5052_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5054_mul_val:
dtype: fp32
shape: [1]
-
+
'770':
dtype: fp32
shape: [5120]
-
+
'771':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3558_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2771_other:
dtype: fp32
shape: [1]
-
+
'772':
dtype: fp32
shape: [5120]
-
+
'773':
dtype: fp32
shape: [5120]
-
+
'774':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3559_bias:
dtype: fp32
shape: [13824]
-
+
'775':
dtype: fp32
shape: [5120]
-
+
'776':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3560_bias:
dtype: fp32
shape: [13824]
-
+
'777':
dtype: fp32
shape: [13824]
-
+
'778':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3561_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2772_other:
dtype: fp32
shape: [1]
-
+
'779':
dtype: fp32
shape: [5120]
-
+
'780':
dtype: fp32
shape: [5120]
-
+
'781':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3562_bias:
dtype: fp32
shape: [5120]
-
+
'782':
dtype: fp32
shape: [5120]
-
+
'783':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3563_bias:
dtype: fp32
shape: [5120]
-
+
'784':
dtype: fp32
shape: [5120]
-
+
'785':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3564_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5084_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5086_mul_val:
dtype: fp32
shape: [1]
-
+
'786':
dtype: fp32
shape: [5120]
-
+
'787':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3565_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2784_other:
dtype: fp32
shape: [1]
-
+
'788':
dtype: fp32
shape: [5120]
-
+
'789':
dtype: fp32
shape: [5120]
-
+
'790':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3566_bias:
dtype: fp32
shape: [13824]
-
+
'791':
dtype: fp32
shape: [5120]
-
+
'792':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3567_bias:
dtype: fp32
shape: [13824]
-
+
'793':
dtype: fp32
shape: [13824]
-
+
'794':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3568_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2785_other:
dtype: fp32
shape: [1]
-
+
'795':
dtype: fp32
shape: [5120]
-
+
'796':
dtype: fp32
shape: [5120]
-
+
'797':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3569_bias:
dtype: fp32
shape: [5120]
-
+
'798':
dtype: fp32
shape: [5120]
-
+
'799':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3570_bias:
dtype: fp32
shape: [5120]
-
+
'800':
dtype: fp32
shape: [5120]
-
+
'801':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3571_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5116_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5118_mul_val:
dtype: fp32
shape: [1]
-
+
'802':
dtype: fp32
shape: [5120]
-
+
'803':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3572_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2797_other:
dtype: fp32
shape: [1]
-
+
'804':
dtype: fp32
shape: [5120]
-
+
'805':
dtype: fp32
shape: [5120]
-
+
'806':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3573_bias:
dtype: fp32
shape: [13824]
-
+
'807':
dtype: fp32
shape: [5120]
-
+
'808':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3574_bias:
dtype: fp32
shape: [13824]
-
+
'809':
dtype: fp32
shape: [13824]
-
+
'810':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3575_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2798_other:
dtype: fp32
shape: [1]
-
+
'811':
dtype: fp32
shape: [5120]
-
+
'812':
dtype: fp32
shape: [5120]
-
+
'813':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3576_bias:
dtype: fp32
shape: [5120]
-
+
'814':
dtype: fp32
shape: [5120]
-
+
'815':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3577_bias:
dtype: fp32
shape: [5120]
-
+
'816':
dtype: fp32
shape: [5120]
-
+
'817':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3578_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5148_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5150_mul_val:
dtype: fp32
shape: [1]
-
+
'818':
dtype: fp32
shape: [5120]
-
+
'819':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3579_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2810_other:
dtype: fp32
shape: [1]
-
+
'820':
dtype: fp32
shape: [5120]
-
+
'821':
dtype: fp32
shape: [5120]
-
+
'822':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3580_bias:
dtype: fp32
shape: [13824]
-
+
'823':
dtype: fp32
shape: [5120]
-
+
'824':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3581_bias:
dtype: fp32
shape: [13824]
-
+
'825':
dtype: fp32
shape: [13824]
-
+
'826':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3582_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2811_other:
dtype: fp32
shape: [1]
-
+
'827':
dtype: fp32
shape: [5120]
-
+
'828':
dtype: fp32
shape: [5120]
-
+
'829':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3583_bias:
dtype: fp32
shape: [5120]
-
+
'830':
dtype: fp32
shape: [5120]
-
+
'831':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3584_bias:
dtype: fp32
shape: [5120]
-
+
'832':
dtype: fp32
shape: [5120]
-
+
'833':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3585_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5180_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5182_mul_val:
dtype: fp32
shape: [1]
-
+
'834':
dtype: fp32
shape: [5120]
-
+
'835':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3586_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2823_other:
dtype: fp32
shape: [1]
-
+
'836':
dtype: fp32
shape: [5120]
-
+
'837':
dtype: fp32
shape: [5120]
-
+
'838':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3587_bias:
dtype: fp32
shape: [13824]
-
+
'839':
dtype: fp32
shape: [5120]
-
+
'840':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3588_bias:
dtype: fp32
shape: [13824]
-
+
'841':
dtype: fp32
shape: [13824]
-
+
'842':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3589_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2824_other:
dtype: fp32
shape: [1]
-
+
'843':
dtype: fp32
shape: [5120]
-
+
'844':
dtype: fp32
shape: [5120]
-
+
'845':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3590_bias:
dtype: fp32
shape: [5120]
-
+
'846':
dtype: fp32
shape: [5120]
-
+
'847':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3591_bias:
dtype: fp32
shape: [5120]
-
+
'848':
dtype: fp32
shape: [5120]
-
+
'849':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3592_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5212_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5214_mul_val:
dtype: fp32
shape: [1]
-
+
'850':
dtype: fp32
shape: [5120]
-
+
'851':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3593_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2836_other:
dtype: fp32
shape: [1]
-
+
'852':
dtype: fp32
shape: [5120]
-
+
'853':
dtype: fp32
shape: [5120]
-
+
'854':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3594_bias:
dtype: fp32
shape: [13824]
-
+
'855':
dtype: fp32
shape: [5120]
-
+
'856':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3595_bias:
dtype: fp32
shape: [13824]
-
+
'857':
dtype: fp32
shape: [13824]
-
+
'858':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3596_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2837_other:
dtype: fp32
shape: [1]
-
+
'859':
dtype: fp32
shape: [5120]
-
+
'860':
dtype: fp32
shape: [5120]
-
+
'861':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3597_bias:
dtype: fp32
shape: [5120]
-
+
'862':
dtype: fp32
shape: [5120]
-
+
'863':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3598_bias:
dtype: fp32
shape: [5120]
-
+
'864':
dtype: fp32
shape: [5120]
-
+
'865':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3599_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5244_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5246_mul_val:
dtype: fp32
shape: [1]
-
+
'866':
dtype: fp32
shape: [5120]
-
+
'867':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3600_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2849_other:
dtype: fp32
shape: [1]
-
+
'868':
dtype: fp32
shape: [5120]
-
+
'869':
dtype: fp32
shape: [5120]
-
+
'870':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3601_bias:
dtype: fp32
shape: [13824]
-
+
'871':
dtype: fp32
shape: [5120]
-
+
'872':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3602_bias:
dtype: fp32
shape: [13824]
-
+
'873':
dtype: fp32
shape: [13824]
-
+
'874':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3603_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2850_other:
dtype: fp32
shape: [1]
-
+
'875':
dtype: fp32
shape: [5120]
-
+
'876':
dtype: fp32
shape: [5120]
-
+
'877':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3604_bias:
dtype: fp32
shape: [5120]
-
+
'878':
dtype: fp32
shape: [5120]
-
+
'879':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3605_bias:
dtype: fp32
shape: [5120]
-
+
'880':
dtype: fp32
shape: [5120]
-
+
'881':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3606_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5276_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5278_mul_val:
dtype: fp32
shape: [1]
-
+
'882':
dtype: fp32
shape: [5120]
-
+
'883':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3607_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2862_other:
dtype: fp32
shape: [1]
-
+
'884':
dtype: fp32
shape: [5120]
-
+
'885':
dtype: fp32
shape: [5120]
-
+
'886':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3608_bias:
dtype: fp32
shape: [13824]
-
+
'887':
dtype: fp32
shape: [5120]
-
+
'888':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3609_bias:
dtype: fp32
shape: [13824]
-
+
'889':
dtype: fp32
shape: [13824]
-
+
'890':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3610_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2863_other:
dtype: fp32
shape: [1]
-
+
'891':
dtype: fp32
shape: [5120]
-
+
'892':
dtype: fp32
shape: [5120]
-
+
'893':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3611_bias:
dtype: fp32
shape: [5120]
-
+
'894':
dtype: fp32
shape: [5120]
-
+
'895':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3612_bias:
dtype: fp32
shape: [5120]
-
+
'896':
dtype: fp32
shape: [5120]
-
+
'897':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3613_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5308_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5310_mul_val:
dtype: fp32
shape: [1]
-
+
'898':
dtype: fp32
shape: [5120]
-
+
'899':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3614_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2875_other:
dtype: fp32
shape: [1]
-
+
'900':
dtype: fp32
shape: [5120]
-
+
'901':
dtype: fp32
shape: [5120]
-
+
'902':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3615_bias:
dtype: fp32
shape: [13824]
-
+
'903':
dtype: fp32
shape: [5120]
-
+
'904':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3616_bias:
dtype: fp32
shape: [13824]
-
+
'905':
dtype: fp32
shape: [13824]
-
+
'906':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3617_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2876_other:
dtype: fp32
shape: [1]
-
+
'907':
dtype: fp32
shape: [5120]
-
+
'908':
dtype: fp32
shape: [5120]
-
+
'909':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3618_bias:
dtype: fp32
shape: [5120]
-
+
'910':
dtype: fp32
shape: [5120]
-
+
'911':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3619_bias:
dtype: fp32
shape: [5120]
-
+
'912':
dtype: fp32
shape: [5120]
-
+
'913':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3620_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5340_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5342_mul_val:
dtype: fp32
shape: [1]
-
+
'914':
dtype: fp32
shape: [5120]
-
+
'915':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3621_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2888_other:
dtype: fp32
shape: [1]
-
+
'916':
dtype: fp32
shape: [5120]
-
+
'917':
dtype: fp32
shape: [5120]
-
+
'918':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3622_bias:
dtype: fp32
shape: [13824]
-
+
'919':
dtype: fp32
shape: [5120]
-
+
'920':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3623_bias:
dtype: fp32
shape: [13824]
-
+
'921':
dtype: fp32
shape: [13824]
-
+
'922':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3624_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2889_other:
dtype: fp32
shape: [1]
-
+
'923':
dtype: fp32
shape: [5120]
-
+
'924':
dtype: fp32
shape: [5120]
-
+
'925':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3625_bias:
dtype: fp32
shape: [5120]
-
+
'926':
dtype: fp32
shape: [5120]
-
+
'927':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3626_bias:
dtype: fp32
shape: [5120]
-
+
'928':
dtype: fp32
shape: [5120]
-
+
'929':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3627_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5372_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5374_mul_val:
dtype: fp32
shape: [1]
-
+
'930':
dtype: fp32
shape: [5120]
-
+
'931':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3628_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2901_other:
dtype: fp32
shape: [1]
-
+
'932':
dtype: fp32
shape: [5120]
-
+
'933':
dtype: fp32
shape: [5120]
-
+
'934':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3629_bias:
dtype: fp32
shape: [13824]
-
+
'935':
dtype: fp32
shape: [5120]
-
+
'936':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3630_bias:
dtype: fp32
shape: [13824]
-
+
'937':
dtype: fp32
shape: [13824]
-
+
'938':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3631_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2902_other:
dtype: fp32
shape: [1]
-
+
'939':
dtype: fp32
shape: [5120]
-
+
'940':
dtype: fp32
shape: [5120]
-
+
'941':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3632_bias:
dtype: fp32
shape: [5120]
-
+
'942':
dtype: fp32
shape: [5120]
-
+
'943':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3633_bias:
dtype: fp32
shape: [5120]
-
+
'944':
dtype: fp32
shape: [5120]
-
+
'945':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3634_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5404_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5406_mul_val:
dtype: fp32
shape: [1]
-
+
'946':
dtype: fp32
shape: [5120]
-
+
'947':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3635_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2914_other:
dtype: fp32
shape: [1]
-
+
'948':
dtype: fp32
shape: [5120]
-
+
'949':
dtype: fp32
shape: [5120]
-
+
'950':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3636_bias:
dtype: fp32
shape: [13824]
-
+
'951':
dtype: fp32
shape: [5120]
-
+
'952':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3637_bias:
dtype: fp32
shape: [13824]
-
+
'953':
dtype: fp32
shape: [13824]
-
+
'954':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3638_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2915_other:
dtype: fp32
shape: [1]
-
+
'955':
dtype: fp32
shape: [5120]
-
+
'956':
dtype: fp32
shape: [5120]
-
+
'957':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3639_bias:
dtype: fp32
shape: [5120]
-
+
'958':
dtype: fp32
shape: [5120]
-
+
'959':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3640_bias:
dtype: fp32
shape: [5120]
-
+
'960':
dtype: fp32
shape: [5120]
-
+
'961':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3641_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5436_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5438_mul_val:
dtype: fp32
shape: [1]
-
+
'962':
dtype: fp32
shape: [5120]
-
+
'963':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3642_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2927_other:
dtype: fp32
shape: [1]
-
+
'964':
dtype: fp32
shape: [5120]
-
+
'965':
dtype: fp32
shape: [5120]
-
+
'966':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3643_bias:
dtype: fp32
shape: [13824]
-
+
'967':
dtype: fp32
shape: [5120]
-
+
'968':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3644_bias:
dtype: fp32
shape: [13824]
-
+
'969':
dtype: fp32
shape: [13824]
-
+
'970':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3645_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2928_other:
dtype: fp32
shape: [1]
-
+
'971':
dtype: fp32
shape: [5120]
-
+
'972':
dtype: fp32
shape: [5120]
-
+
'973':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3646_bias:
dtype: fp32
shape: [5120]
-
+
'974':
dtype: fp32
shape: [5120]
-
+
'975':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3647_bias:
dtype: fp32
shape: [5120]
-
+
'976':
dtype: fp32
shape: [5120]
-
+
'977':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3648_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5468_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5470_mul_val:
dtype: fp32
shape: [1]
-
+
'978':
dtype: fp32
shape: [5120]
-
+
'979':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3649_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2940_other:
dtype: fp32
shape: [1]
-
+
'980':
dtype: fp32
shape: [5120]
-
+
'981':
dtype: fp32
shape: [5120]
-
+
'982':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3650_bias:
dtype: fp32
shape: [13824]
-
+
'983':
dtype: fp32
shape: [5120]
-
+
'984':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3651_bias:
dtype: fp32
shape: [13824]
-
+
'985':
dtype: fp32
shape: [13824]
-
+
'986':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3652_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2941_other:
dtype: fp32
shape: [1]
-
+
'987':
dtype: fp32
shape: [5120]
-
+
'988':
dtype: fp32
shape: [5120]
-
+
'989':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3653_bias:
dtype: fp32
shape: [5120]
-
+
'990':
dtype: fp32
shape: [5120]
-
+
'991':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3654_bias:
dtype: fp32
shape: [5120]
-
+
'992':
dtype: fp32
shape: [5120]
-
+
'993':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3655_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5500_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5502_mul_val:
dtype: fp32
shape: [1]
-
+
'994':
dtype: fp32
shape: [5120]
-
+
'995':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3656_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2953_other:
dtype: fp32
shape: [1]
-
+
'996':
dtype: fp32
shape: [5120]
-
+
'997':
dtype: fp32
shape: [5120]
-
+
'998':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3657_bias:
dtype: fp32
shape: [13824]
-
+
'999':
dtype: fp32
shape: [5120]
-
+
'1000':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3658_bias:
dtype: fp32
shape: [13824]
-
+
'1001':
dtype: fp32
shape: [13824]
-
+
'1002':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3659_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2954_other:
dtype: fp32
shape: [1]
-
+
'1003':
dtype: fp32
shape: [5120]
-
+
'1004':
dtype: fp32
shape: [5120]
-
+
'1005':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3660_bias:
dtype: fp32
shape: [5120]
-
+
'1006':
dtype: fp32
shape: [5120]
-
+
'1007':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3661_bias:
dtype: fp32
shape: [5120]
-
+
'1008':
dtype: fp32
shape: [5120]
-
+
'1009':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3662_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5532_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5534_mul_val:
dtype: fp32
shape: [1]
-
+
'1010':
dtype: fp32
shape: [5120]
-
+
'1011':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3663_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2966_other:
dtype: fp32
shape: [1]
-
+
'1012':
dtype: fp32
shape: [5120]
-
+
'1013':
dtype: fp32
shape: [5120]
-
+
'1014':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3664_bias:
dtype: fp32
shape: [13824]
-
+
'1015':
dtype: fp32
shape: [5120]
-
+
'1016':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3665_bias:
dtype: fp32
shape: [13824]
-
+
'1017':
dtype: fp32
shape: [13824]
-
+
'1018':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3666_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2967_other:
dtype: fp32
shape: [1]
-
+
'1019':
dtype: fp32
shape: [5120]
-
+
'1020':
dtype: fp32
shape: [5120]
-
+
'1021':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3667_bias:
dtype: fp32
shape: [5120]
-
+
'1022':
dtype: fp32
shape: [5120]
-
+
'1023':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3668_bias:
dtype: fp32
shape: [5120]
-
+
'1024':
dtype: fp32
shape: [5120]
-
+
'1025':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3669_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5564_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5566_mul_val:
dtype: fp32
shape: [1]
-
+
'1026':
dtype: fp32
shape: [5120]
-
+
'1027':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3670_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2979_other:
dtype: fp32
shape: [1]
-
+
'1028':
dtype: fp32
shape: [5120]
-
+
'1029':
dtype: fp32
shape: [5120]
-
+
'1030':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3671_bias:
dtype: fp32
shape: [13824]
-
+
'1031':
dtype: fp32
shape: [5120]
-
+
'1032':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3672_bias:
dtype: fp32
shape: [13824]
-
+
'1033':
dtype: fp32
shape: [13824]
-
+
'1034':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3673_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2980_other:
dtype: fp32
shape: [1]
-
+
'1035':
dtype: fp32
shape: [5120]
-
+
'1036':
dtype: fp32
shape: [5120]
-
+
'1037':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3674_bias:
dtype: fp32
shape: [5120]
-
+
'1038':
dtype: fp32
shape: [5120]
-
+
'1039':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3675_bias:
dtype: fp32
shape: [5120]
-
+
'1040':
dtype: fp32
shape: [5120]
-
+
'1041':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3676_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5596_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5598_mul_val:
dtype: fp32
shape: [1]
-
+
'1042':
dtype: fp32
shape: [5120]
-
+
'1043':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3677_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2992_other:
dtype: fp32
shape: [1]
-
+
'1044':
dtype: fp32
shape: [5120]
-
+
'1045':
dtype: fp32
shape: [5120]
-
+
'1046':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3678_bias:
dtype: fp32
shape: [13824]
-
+
'1047':
dtype: fp32
shape: [5120]
-
+
'1048':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3679_bias:
dtype: fp32
shape: [13824]
-
+
'1049':
dtype: fp32
shape: [13824]
-
+
'1050':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3680_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_2993_other:
dtype: fp32
shape: [1]
-
+
'1051':
dtype: fp32
shape: [5120]
-
+
'1052':
dtype: fp32
shape: [5120]
-
+
'1053':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3681_bias:
dtype: fp32
shape: [5120]
-
+
'1054':
dtype: fp32
shape: [5120]
-
+
'1055':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3682_bias:
dtype: fp32
shape: [5120]
-
+
'1056':
dtype: fp32
shape: [5120]
-
+
'1057':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3683_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5628_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5630_mul_val:
dtype: fp32
shape: [1]
-
+
'1058':
dtype: fp32
shape: [5120]
-
+
'1059':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3684_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3005_other:
dtype: fp32
shape: [1]
-
+
'1060':
dtype: fp32
shape: [5120]
-
+
'1061':
dtype: fp32
shape: [5120]
-
+
'1062':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3685_bias:
dtype: fp32
shape: [13824]
-
+
'1063':
dtype: fp32
shape: [5120]
-
+
'1064':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3686_bias:
dtype: fp32
shape: [13824]
-
+
'1065':
dtype: fp32
shape: [13824]
-
+
'1066':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3687_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3006_other:
dtype: fp32
shape: [1]
-
+
'1067':
dtype: fp32
shape: [5120]
-
+
'1068':
dtype: fp32
shape: [5120]
-
+
'1069':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3688_bias:
dtype: fp32
shape: [5120]
-
+
'1070':
dtype: fp32
shape: [5120]
-
+
'1071':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3689_bias:
dtype: fp32
shape: [5120]
-
+
'1072':
dtype: fp32
shape: [5120]
-
+
'1073':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3690_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5660_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5662_mul_val:
dtype: fp32
shape: [1]
-
+
'1074':
dtype: fp32
shape: [5120]
-
+
'1075':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3691_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3018_other:
dtype: fp32
shape: [1]
-
+
'1076':
dtype: fp32
shape: [5120]
-
+
'1077':
dtype: fp32
shape: [5120]
-
+
'1078':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3692_bias:
dtype: fp32
shape: [13824]
-
+
'1079':
dtype: fp32
shape: [5120]
-
+
'1080':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3693_bias:
dtype: fp32
shape: [13824]
-
+
'1081':
dtype: fp32
shape: [13824]
-
+
'1082':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3694_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3019_other:
dtype: fp32
shape: [1]
-
+
'1083':
dtype: fp32
shape: [5120]
-
+
'1084':
dtype: fp32
shape: [5120]
-
+
'1085':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3695_bias:
dtype: fp32
shape: [5120]
-
+
'1086':
dtype: fp32
shape: [5120]
-
+
'1087':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3696_bias:
dtype: fp32
shape: [5120]
-
+
'1088':
dtype: fp32
shape: [5120]
-
+
'1089':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3697_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5692_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5694_mul_val:
dtype: fp32
shape: [1]
-
+
'1090':
dtype: fp32
shape: [5120]
-
+
'1091':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3698_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3031_other:
dtype: fp32
shape: [1]
-
+
'1092':
dtype: fp32
shape: [5120]
-
+
'1093':
dtype: fp32
shape: [5120]
-
+
'1094':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3699_bias:
dtype: fp32
shape: [13824]
-
+
'1095':
dtype: fp32
shape: [5120]
-
+
'1096':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3700_bias:
dtype: fp32
shape: [13824]
-
+
'1097':
dtype: fp32
shape: [13824]
-
+
'1098':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3701_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3032_other:
dtype: fp32
shape: [1]
-
+
'1099':
dtype: fp32
shape: [5120]
-
+
'1100':
dtype: fp32
shape: [5120]
-
+
'1101':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3702_bias:
dtype: fp32
shape: [5120]
-
+
'1102':
dtype: fp32
shape: [5120]
-
+
'1103':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3703_bias:
dtype: fp32
shape: [5120]
-
+
'1104':
dtype: fp32
shape: [5120]
-
+
'1105':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3704_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5724_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5726_mul_val:
dtype: fp32
shape: [1]
-
+
'1106':
dtype: fp32
shape: [5120]
-
+
'1107':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3705_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3044_other:
dtype: fp32
shape: [1]
-
+
'1108':
dtype: fp32
shape: [5120]
-
+
'1109':
dtype: fp32
shape: [5120]
-
+
'1110':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3706_bias:
dtype: fp32
shape: [13824]
-
+
'1111':
dtype: fp32
shape: [5120]
-
+
'1112':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3707_bias:
dtype: fp32
shape: [13824]
-
+
'1113':
dtype: fp32
shape: [13824]
-
+
'1114':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3708_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3045_other:
dtype: fp32
shape: [1]
-
+
'1115':
dtype: fp32
shape: [5120]
-
+
'1116':
dtype: fp32
shape: [5120]
-
+
'1117':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3709_bias:
dtype: fp32
shape: [5120]
-
+
'1118':
dtype: fp32
shape: [5120]
-
+
'1119':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3710_bias:
dtype: fp32
shape: [5120]
-
+
'1120':
dtype: fp32
shape: [5120]
-
+
'1121':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3711_bias:
dtype: fp32
shape: [5120]
-
+
aten::neg_5756_mul_val:
dtype: fp32
shape: [1]
-
+
aten::neg_5758_mul_val:
dtype: fp32
shape: [1]
-
+
'1122':
dtype: fp32
shape: [5120]
-
+
'1123':
dtype: s8
shape: [5120, 5120]
-
+
aten::linear_3712_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3057_other:
dtype: fp32
shape: [1]
-
+
'1124':
dtype: fp32
shape: [5120]
-
+
'1125':
dtype: fp32
shape: [5120]
-
+
'1126':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3713_bias:
dtype: fp32
shape: [13824]
-
+
'1127':
dtype: fp32
shape: [5120]
-
+
'1128':
dtype: s8
shape: [13824, 5120]
-
+
aten::linear_3714_bias:
dtype: fp32
shape: [13824]
-
+
'1129':
dtype: fp32
shape: [13824]
-
+
'1130':
dtype: s8
shape: [5120, 13824]
-
+
aten::linear_3715_bias:
dtype: fp32
shape: [5120]
-
+
aten::pow_3058_other:
dtype: fp32
shape: [1]
-
+
'1131':
dtype: fp32
shape: [5120]
-
+
'1132':
dtype: fp32
shape: [5120]
-
+
'1133':
dtype: s8
shape: [32000, 5120]
-
+
aten::linear_3716_bias:
dtype: fp32
shape: [32000]
-
+
aten::size_0:
type: Shape
input:
@@ -32138,7 +32138,7 @@ def test_1(self):
llamagraph = p_fusion(llamagraph)
newlen = len(llamagraph.nodes)
self.assertTrue(oldlen != newlen)
-
+
if __name__ == "__main__":
unittest.main()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul.py
index d5d5a581126..3faf4261e40 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul.py
@@ -48,11 +48,11 @@ def test_1(self):
example_in = torch.rand(3, 256)
example_in2 = torch.rand(256, 10)
traced_model = torch.jit.trace(n, (example_in, example_in2))
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
# torch.onnx.export(n, example_in, '{}.onnx'.format(file_name))
ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul_int8.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul_int8.py
index 3b978c8f799..13f13d9a50e 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul_int8.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_matmul_int8.py
@@ -65,11 +65,11 @@ def test_1(self):
convert_model = convert(prepared_model)
traced_model = torch.jit.trace(convert_model, (example_in, example_in2))
print(traced_model.inlined_graph)
-
+
torch.jit.freeze(traced_model.eval())
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_neox_rotary_pos_emb.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_neox_rotary_pos_emb.py
index b55e2e769e2..42233b6646f 100755
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_neox_rotary_pos_emb.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_neox_rotary_pos_emb.py
@@ -751,4 +751,4 @@ def test_1(self):
self.assertTrue(oldlen != newlen)
if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_padding_seq.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_padding_seq.py
index a81230086a8..70c90dca3c1 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_padding_seq.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_padding_seq.py
@@ -55,7 +55,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, (example_in, example_in2))
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in, example_in2).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_permute.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_permute.py
index 2a491999343..76899e2e680 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_permute.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_permute.py
@@ -48,11 +48,11 @@ def test_1(self):
n = Net()
example_in = torch.randn(20, 5, 10)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
torch.onnx.export(n, example_in, '{}.onnx'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_pow.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_pow.py
index 03988318ed0..3d87a08cf1b 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_pow.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_pow.py
@@ -50,7 +50,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, example_in)
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_size.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_size.py
index eb1ba91698d..faf67d1f2f9 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_size.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_size.py
@@ -47,10 +47,10 @@ def test_1(self):
n = Net()
example_in = torch.rand(3, 30)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_slice.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_slice.py
index a67941f0645..54e9c00b4f2 100755
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_slice.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_slice.py
@@ -47,10 +47,10 @@ def test_1(self):
n = Net()
example_in = torch.rand(4, 64, 96, 300)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_softmax.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_softmax.py
index a059d3ab542..10ef0a7398b 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_softmax.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_softmax.py
@@ -51,7 +51,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, example_in)
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_tanh.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_tanh.py
index b3cf3a2c8a2..1b2d860166b 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_tanh.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_tanh.py
@@ -51,7 +51,7 @@ def test_1(self):
traced_model = torch.jit.trace(n, example_in)
torch.jit.save(traced_model, '{}.pt'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_transpose.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_transpose.py
index 16014b72070..f9f519680f0 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_transpose.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_transpose.py
@@ -48,12 +48,12 @@ def test_1(self):
n = Net()
example_in = torch.rand(3, 4, 5)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
# torch.onnx.export(n, example_in, '{}.onnx'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
print(ref_out.shape)
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
@@ -68,12 +68,12 @@ def test_2(self):
n = Net()
example_in = torch.rand(3, 4)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
# torch.onnx.export(n, example_in, '{}.onnx'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
print(ref_out.shape)
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_two_linear.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_two_linear.py
index 431ddc0c80e..c4d63257b86 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_two_linear.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_two_linear.py
@@ -80,7 +80,7 @@ def test_per_tensor(self):
prepared_model(example_in)
convert_model = convert(prepared_model)
traced_model = torch.jit.trace(convert_model, example_in)
-
+
torch.jit.freeze(traced_model.eval())
torch.jit.save(traced_model, '{}.pt'.format(file_name))
@@ -91,7 +91,7 @@ def test_per_tensor(self):
newgraph = Graph()
newgraph.graph_init(file_name + '/conf.yaml', file_name + '/model.bin')
out = newgraph.inference([example_in.numpy()])
-
+
self.assertTrue(cmpData(ref_out, [*out.values()][0]) < 0.01)
os.remove('{}.pt'.format(file_name))
shutil.rmtree(file_name)
diff --git a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_unsqueeze.py b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_unsqueeze.py
index b957431c251..ae087a164ea 100644
--- a/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_unsqueeze.py
+++ b/intel_extension_for_transformers/llm/runtime/deprecated/test/pytest/torch_test/test_unsqueeze.py
@@ -47,11 +47,11 @@ def test_1(self):
n = Net()
example_in = torch.randn(1, 77, 77)
traced_model = torch.jit.trace(n, example_in)
-
+
torch.jit.save(traced_model, '{}.pt'.format(file_name))
# torch.onnx.export(n, example_in, '{}.onnx'.format(file_name))
ref_out = traced_model(example_in).detach().numpy()
-
+
graph = compile('{}.pt'.format(file_name))
graph.save(file_name)
newgraph = Graph()
diff --git a/intel_extension_for_transformers/llm/runtime/graph/requirements.txt b/intel_extension_for_transformers/llm/runtime/graph/requirements.txt
index 1601a33c0d7..92b9b33437b 100644
--- a/intel_extension_for_transformers/llm/runtime/graph/requirements.txt
+++ b/intel_extension_for_transformers/llm/runtime/graph/requirements.txt
@@ -1,17 +1,17 @@
--extra-index-url https://download.pytorch.org/whl/cpu
-torch==2.1.0+cpu
-transformers
-numpy
-sentencepiece
-protobuf<3.20
-einops
accelerate
-peft
-datasets
-transformers_stream_generator
-tiktoken
-py-cpuinfo
+auto-gptq
cmake
+datasets
+einops
gguf
-auto-gptq
-neural-speed
\ No newline at end of file
+neural-speed
+numpy
+peft
+protobuf<3.20
+py-cpuinfo
+sentencepiece
+tiktoken
+torch==2.1.0+cpu
+transformers
+transformers_stream_generator
diff --git a/intel_extension_for_transformers/llm/runtime/graph/tests/test_llm_runtime.py b/intel_extension_for_transformers/llm/runtime/graph/tests/test_llm_runtime.py
index ea479c281ca..4b9407c1767 100644
--- a/intel_extension_for_transformers/llm/runtime/graph/tests/test_llm_runtime.py
+++ b/intel_extension_for_transformers/llm/runtime/graph/tests/test_llm_runtime.py
@@ -49,7 +49,7 @@ def test_llm_runtime(self):
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
inputs = tokenizer(prompt, return_tensors="pt")
-
+
pt_logits = torch.load("/tf_dataset2/inc-ut/nlptoolkit_ut_model/llama2_pt_logits.pth")[:,-1]
pt_generate_ids = torch.load("/tf_dataset2/inc-ut/nlptoolkit_ut_model/llama2_pt_generate_ids.pth")[0].tolist()
print(tokenizer.decode(pt_generate_ids))
@@ -70,7 +70,7 @@ def test_llm_runtime(self):
# "jblas_int8": WeightOnlyQuantConfig(compute_dtype="bf16", weight_dtype="int8"),
}
for config_type in woq_configs:
- itrex_model = AutoModel.from_pretrained(model_name, quantization_config=woq_configs[config_type],
+ itrex_model = AutoModel.from_pretrained(model_name, quantization_config=woq_configs[config_type],
use_llm_runtime=True, trust_remote_code=True)
itrex_logits = itrex_model(inputs.input_ids)
print(config_type, cmpData(pt_logits.detach().numpy().flatten(), itrex_logits.flatten()))
diff --git a/intel_extension_for_transformers/llm/utils/__init__.py b/intel_extension_for_transformers/llm/utils/__init__.py
index 407b29e53be..18896e7b549 100644
--- a/intel_extension_for_transformers/llm/utils/__init__.py
+++ b/intel_extension_for_transformers/llm/utils/__init__.py
@@ -14,4 +14,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
diff --git a/intel_extension_for_transformers/llm/utils/mpt_trace.py b/intel_extension_for_transformers/llm/utils/mpt_trace.py
index f316b9e7ab4..5634025a276 100644
--- a/intel_extension_for_transformers/llm/utils/mpt_trace.py
+++ b/intel_extension_for_transformers/llm/utils/mpt_trace.py
@@ -85,4 +85,4 @@ def forward(
logits=outputs["logits"], past_key_values=outputs["past_key_values"] if self.use_cache else None
)
- return outputs
\ No newline at end of file
+ return outputs
diff --git a/intel_extension_for_transformers/neural_chat/__init__.py b/intel_extension_for_transformers/neural_chat/__init__.py
index 3a3d819f5c4..263aa977940 100644
--- a/intel_extension_for_transformers/neural_chat/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/__init__.py
@@ -29,4 +29,3 @@
from .server.neuralchat_server import NeuralChatServerExecutor
from .server.neuralchat_client import TextChatClientExecutor, VoiceChatClientExecutor, FinetuningClientExecutor
from .plugins import plugins
-
diff --git a/intel_extension_for_transformers/neural_chat/cli/README.md b/intel_extension_for_transformers/neural_chat/cli/README.md
index 6a809d2c2fb..cec1cd7a3a7 100644
--- a/intel_extension_for_transformers/neural_chat/cli/README.md
+++ b/intel_extension_for_transformers/neural_chat/cli/README.md
@@ -33,4 +33,3 @@ neuralchat help
voicechat = TextVoiceChatExecutor()
voicechat(query="../../assets/audio/say_hello.wav", output_audio_path="response.wav")
```
-
diff --git a/intel_extension_for_transformers/neural_chat/cli/base_executor.py b/intel_extension_for_transformers/neural_chat/cli/base_executor.py
index c841ae06233..21bd560cc94 100644
--- a/intel_extension_for_transformers/neural_chat/cli/base_executor.py
+++ b/intel_extension_for_transformers/neural_chat/cli/base_executor.py
@@ -48,4 +48,4 @@ def __call__(self, *arg, **kwargs):
"""
Python API to call an executor.
"""
- pass
\ No newline at end of file
+ pass
diff --git a/intel_extension_for_transformers/neural_chat/config.py b/intel_extension_for_transformers/neural_chat/config.py
index 2e00efcbb74..f2c41a551f0 100644
--- a/intel_extension_for_transformers/neural_chat/config.py
+++ b/intel_extension_for_transformers/neural_chat/config.py
@@ -366,7 +366,7 @@ class TTSModelArguments:
step: int = field(default=0, metadata={"help": "TTS model step."})
warmup_step: int = field(default=0, metadata={"help": "TTS model warmup step."})
learning_rate: float = field(default=1e-5, metadata={"help": "Learning rate."})
-
+
@dataclass
class BaseFinetuningConfig:
model_args: ModelArguments
diff --git a/intel_extension_for_transformers/neural_chat/config_logging.py b/intel_extension_for_transformers/neural_chat/config_logging.py
index 6763a9c9ee3..fc37ee36d2c 100644
--- a/intel_extension_for_transformers/neural_chat/config_logging.py
+++ b/intel_extension_for_transformers/neural_chat/config_logging.py
@@ -50,4 +50,3 @@ def configure_logging(log_file="app.log", log_level=logging.INFO):
logger.addHandler(console_handler)
return logger
-
diff --git a/intel_extension_for_transformers/neural_chat/docker/finetuning/README.md b/intel_extension_for_transformers/neural_chat/docker/finetuning/README.md
index 6fb46da43ea..711d788fa9d 100644
--- a/intel_extension_for_transformers/neural_chat/docker/finetuning/README.md
+++ b/intel_extension_for_transformers/neural_chat/docker/finetuning/README.md
@@ -326,4 +326,3 @@ Where the `--dataset_concatenation` argument is a way to vastly accelerate the f
For finetuning on SPR, add `--bf16` argument will speedup the finetuning process without the loss of model's performance.
You could also indicate `--peft` to switch peft method in P-tuning, Prefix tuning, Prompt tuning, LLama Adapter, LoRA,
see https://github.com/huggingface/peft. Note for MPT, only LoRA is supported.
-
diff --git a/intel_extension_for_transformers/neural_chat/docker/tgi_serving/README.md b/intel_extension_for_transformers/neural_chat/docker/tgi_serving/README.md
index 331534d2eac..98e8974519d 100644
--- a/intel_extension_for_transformers/neural_chat/docker/tgi_serving/README.md
+++ b/intel_extension_for_transformers/neural_chat/docker/tgi_serving/README.md
@@ -34,5 +34,3 @@ curl ${your_ip}:${your_port}/v1/tgi/generate \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-H 'Content-Type: application/json'
```
-
-
diff --git a/intel_extension_for_transformers/neural_chat/docs/advanced_features.md b/intel_extension_for_transformers/neural_chat/docs/advanced_features.md
index b2e11d8c2f1..ec5eb8751ab 100644
--- a/intel_extension_for_transformers/neural_chat/docs/advanced_features.md
+++ b/intel_extension_for_transformers/neural_chat/docs/advanced_features.md
@@ -196,4 +196,3 @@ For detailed fine-tuning instructions, please refer to the documentation below.
[Text-To-Speech (TTS) model finetuning](./examples/finetuning/tts/README.md)
And NeuralChat also provides Docker file tailored for easy fine-tuning. Explore details in [finetuning with Docker](./docker/finetuning/README.md).
-
diff --git a/intel_extension_for_transformers/neural_chat/docs/neuralchat_api.md b/intel_extension_for_transformers/neural_chat/docs/neuralchat_api.md
index 57c738892b9..5bcd76cb357 100644
--- a/intel_extension_for_transformers/neural_chat/docs/neuralchat_api.md
+++ b/intel_extension_for_transformers/neural_chat/docs/neuralchat_api.md
@@ -108,4 +108,3 @@ curl http://localhost:80/v1/completions \
"temperature": 0.5
}'
```
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/README.md b/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/README.md
index 5488ea43938..93ddd5954a8 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/README.md
@@ -4,5 +4,3 @@ Welcome to Photo AI! This example introduces how to deploy the Text Chatbot syst
| ---------------------| --------------------------|
| Backend Setup | [Backend README](./backend/README.md) |
| Frontend Setup | [Frontend README](../../../ui/customized/talking_photo/README.md) |
-
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/backend/photoai.py b/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/backend/photoai.py
index b4e2c17db43..6c277a505d9 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/backend/photoai.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/photo_ai/backend/photoai.py
@@ -26,4 +26,4 @@ def main():
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/README.md b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/README.md
index 5170ef58706..4ffee71f7f2 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/README.md
@@ -75,4 +75,4 @@ nohup bash run.sh &
To call the started audio service, the APIs are listed as follows:
1. http://127.0.0.1:7777/plugin/audio/asr , upload an audio file and return the text contents.
2. http://127.0.0.1:7777/plugin/audio/tts , input text string and return the binary content of the audio.
-3. http://127.0.0.1:7777/plugin/audio/create_embedding, upload an audio file and create an embedding of your voice.
\ No newline at end of file
+3. http://127.0.0.1:7777/plugin/audio/create_embedding, upload an audio file and create an embedding of your voice.
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/audio_service.py b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/audio_service.py
index 0b538bc1777..fc1bd3a09f3 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/audio_service.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/audio/audio_service.py
@@ -26,4 +26,4 @@ def main():
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/image2image_service.py b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/image2image_service.py
index 8c27e9686b5..dbd0a508faf 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/image2image_service.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/image2image_service.py
@@ -26,4 +26,4 @@ def main():
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/prepare_model.py b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/prepare_model.py
index 5977c186e01..9cdf50b6670 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/prepare_model.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/prepare_model.py
@@ -254,7 +254,7 @@ def wrapper(*args, **kwargs):
return output
return wrapper
unet.forward = model_wrapper(unet.forward)
-
+
torch.onnx.export(
unet,
args=(
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/quantization_modules.py b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/quantization_modules.py
index 0470456db31..94f0c0f8043 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/quantization_modules.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/quantization_modules.py
@@ -160,4 +160,4 @@ def load_int8_model(fp32_model, int8_model_path, fake_quantize_model=False):
print('Converted to quantized model.')
if not fake_quantize_model:
int8_model.load_state_dict(torch.load(int8_model_path))
- return int8_model
\ No newline at end of file
+ return int8_model
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/requirements.txt b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/requirements.txt
index 71e156b6c16..580d9b63f8d 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/plugin/image2image/requirements.txt
@@ -1,12 +1,12 @@
-neural-compressor
-transformers
accelerate
datasets >= 1.8.0
-sentencepiece != 0.1.92
-protobuf
-torch==2.1.0
+diffusers==0.12.1
+neural-compressor
onnx>=1.12
onnxruntime==1.13.1
-diffusers==0.12.1
-pytorch_fid
optimum
+protobuf
+pytorch_fid
+sentencepiece != 0.1.92
+torch==2.1.0
+transformers
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.py b/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.py
index 294e83b7a21..0f8079d9948 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.py
@@ -26,4 +26,4 @@ def main():
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.yaml b/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.yaml
index 00f3855bee6..c6b4f04a687 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.yaml
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/rag/askdoc.yaml
@@ -38,4 +38,3 @@ safety_checker:
enable: true
tasks_list: ['textchat', 'retrieval']
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/README.md b/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/README.md
index 1bac6791d2b..8c91d373cd0 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/README.md
@@ -4,4 +4,3 @@ Welcome to Talkingbot! This example introduces how to deploy the Talkingbot syst
| ---------------------| --------------------------|
| Backend Setup | [Backend README](./backend/README.md) |
| Frontend Setup | [Frontend README](../../../../ui/customized/talkingbot/README.md) |
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/backend/talkingbot.py b/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/backend/talkingbot.py
index 5985deb0d0a..1922c744cc9 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/backend/talkingbot.py
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/talkingbot/server/backend/talkingbot.py
@@ -22,4 +22,4 @@ def main():
server_executor(config_file="./talkingbot.yaml", log_file="./talkingbot.log")
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/intel_extension_for_transformers/neural_chat/examples/deployment/textbot/README.md b/intel_extension_for_transformers/neural_chat/examples/deployment/textbot/README.md
index d10d313e1e6..722da719533 100644
--- a/intel_extension_for_transformers/neural_chat/examples/deployment/textbot/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/deployment/textbot/README.md
@@ -6,4 +6,3 @@ Welcome to the Text Chatbot! This example introduces how to deploy the Text Chat
| Frontend Setup | [Frontend README](../../../ui/gradio/basic/README.md) |
You can enhance the capabilities of the Text Chatbot by enabling plugins, such as the cache plugin. This plugin is designed to help you reduce costs effectively by caching query results, resulting in fewer requests and tokens sent to the Language Model service. As a result, it offers superior query throughput compared to standard Language Model services. To deploy a Text Chatbot with caching functionality, please refer to the instructions provided in the [README](./backend_with_cache/README.md) for backend setup.
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/README.md b/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/README.md
index 7fa401327bd..0cd6232a0ae 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/README.md
@@ -77,5 +77,3 @@ We verify DPO training on our finetuned `mpt-7b` model [Intel/neural-chat-7b-v1-
| [mosaicml/mpt-7b-chat](https://huggingface.co/mosaicml/mpt-7b-chat) | 49.95 | 46.5 | 75.55 | 37.60 | 40.17 | ours |
| [Intel/neural-chat-7b-v1-1](https://huggingface.co/Intel/neural-chat-7b-v1-1) | **51.41** | 50.09 | 76.69 | 38.79 | 40.07 | ours |
| **[Intel/neural-chat-7b-v1-1](https://huggingface.co/Intel/neural-chat-7b-v1-1) with DPO** | **52.39** | 51.54 | 76.45 | 39.47| 42.10 | ours |
-
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/dpo_clm.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/dpo_clm.py
index 76ca1bbad9b..54315a73c5f 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/dpo_clm.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/dpo_clm.py
@@ -269,7 +269,7 @@ def return_prompt_and_responses(samples) -> Dict[str, str]:
# model config
config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
torch_dtype = (
- model_args.torch_dtype if model_args.torch_dtype in ["auto", None]
+ model_args.torch_dtype if model_args.torch_dtype in ["auto", None]
else getattr(torch, model_args.torch_dtype)
)
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/requirements.txt b/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/requirements.txt
index 3b5e3f073a0..573855a44e1 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/dpo_pipeline/requirements.txt
@@ -1,10 +1,10 @@
datasets
-torch
-transformers
-sentencepiece
-peft
+einops
evaluate
+fire
nltk
+peft
rouge_score
-einops
-fire
+sentencepiece
+torch
+transformers
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/README.md b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/README.md
index 1d5f6102048..1bec6ac3950 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/README.md
@@ -635,4 +635,3 @@ see https://github.com/huggingface/peft. Note for MPT, only LoRA is supported.
- **rouge related metrics:** the metrics will be calculated when the finetuning task is summarization `--task summarization`
- **human eval (code generation metric):** the metric will be calculated when the finetuning task is code-generation `--task code-generation`
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/finetune_seq2seq.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/finetune_seq2seq.py
index f96e79e88a2..7af300a8c2e 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/finetune_seq2seq.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/finetune_seq2seq.py
@@ -57,4 +57,4 @@ def main():
finetune_model(finetune_cfg)
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/gaudi_spawn.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/gaudi_spawn.py
index 6978f69fa73..9fd57f89886 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/gaudi_spawn.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/gaudi_spawn.py
@@ -110,4 +110,3 @@ def main():
if __name__ == "__main__":
main()
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/requirements.txt b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/requirements.txt
index 38a77749c4c..4701e52a719 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/instruction/requirements.txt
@@ -1,15 +1,15 @@
datasets
-torch==2.1.0
-transformers
-sentencepiece
-peft
+einops
evaluate
+fastapi
nltk
+peft
+pydub
+python-multipart
rouge_score
-einops
+sentencepiece
+shortuuid
+torch==2.1.0
+transformers
uvicorn
yacs
-fastapi
-shortuuid
-pydub
-python-multipart
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/conversation_utils.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/conversation_utils.py
index d8b4704bdef..3ec5da5e046 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/conversation_utils.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/conversation_utils.py
@@ -262,4 +262,3 @@ def dict(self):
"v1": conv_llava_v1,
"plain": conv_llava_plain,
}
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/gaudi_spawn.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/gaudi_spawn.py
index 6978f69fa73..9fd57f89886 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/gaudi_spawn.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/gaudi_spawn.py
@@ -110,4 +110,3 @@ def main():
if __name__ == "__main__":
main()
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/llava_utils.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/llava_utils.py
index 745cde58cff..d38fb918c28 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/llava_utils.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/llava_utils.py
@@ -686,4 +686,3 @@ def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,
return dict(train_dataset=train_dataset,
eval_dataset=None,
data_collator=data_collator)
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/ppo_pipeline/requirements.txt b/intel_extension_for_transformers/neural_chat/examples/finetuning/ppo_pipeline/requirements.txt
index 5365f82d20b..5fae17c3f2c 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/ppo_pipeline/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/ppo_pipeline/requirements.txt
@@ -1,9 +1,9 @@
-transformers
-peft
accelerate
-datasets
bitsandbytes
+datasets
evaluate
-scikit-learn
intel-extension-for-transformers
+peft
+scikit-learn
+transformers
tyro
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/finetune.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/finetune.py
index c073c0f7ae8..53d3ed199da 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/finetune.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/finetune.py
@@ -30,5 +30,3 @@
finetuned_model = tts_fintuner.finetune()
torch.save(finetuned_model, "finetuned_model.pt")
-
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/inference.py b/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/inference.py
index dcb556c8981..223a8fddc50 100644
--- a/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/inference.py
+++ b/intel_extension_for_transformers/neural_chat/examples/finetuning/tts/inference.py
@@ -138,4 +138,3 @@ def correct_number(text):
except Exception as e:
logging.info("Catch exception: %s", e)
logging.info("Restarting\n")
-
\ No newline at end of file
diff --git a/intel_extension_for_transformers/neural_chat/examples/plugins/retrieval/README.md b/intel_extension_for_transformers/neural_chat/examples/plugins/retrieval/README.md
index 8e07e70ee0a..3057b8eee27 100644
--- a/intel_extension_for_transformers/neural_chat/examples/plugins/retrieval/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/plugins/retrieval/README.md
@@ -37,5 +37,3 @@ You can utilize the following script to execute the code on Intel® Xeon SPR pro
conda install jemalloc gperftools -c conda-forge -y
bash run_retrieval_on_cpu.sh
```
-
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/plugins/video/README.md b/intel_extension_for_transformers/neural_chat/examples/plugins/video/README.md
index 4632c939b96..82e7abcec3d 100644
--- a/intel_extension_for_transformers/neural_chat/examples/plugins/video/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/plugins/video/README.md
@@ -31,4 +31,4 @@ neuralchat_server start --config_file face_animation.yaml
## Acknowledgements
-This plugin is mostly adapted from [SadTalker](https://github.com/OpenTalker/SadTalker). We thank the related authors for their great work!
\ No newline at end of file
+This plugin is mostly adapted from [SadTalker](https://github.com/OpenTalker/SadTalker). We thank the related authors for their great work!
diff --git a/intel_extension_for_transformers/neural_chat/examples/plugins/video/main.py b/intel_extension_for_transformers/neural_chat/examples/plugins/video/main.py
index 3316170973b..cebc92c62f8 100644
--- a/intel_extension_for_transformers/neural_chat/examples/plugins/video/main.py
+++ b/intel_extension_for_transformers/neural_chat/examples/plugins/video/main.py
@@ -36,4 +36,3 @@
sadtalker = SadTalker(device=device, checkpoint_dir=checkpoint_dir, bf16=True, p_num=4, enhancer=None, output_video_path=output_video_path)
sadtalker.convert(source_image=source_image, driven_audio=driven_audio)
-
diff --git a/intel_extension_for_transformers/neural_chat/examples/serving/TGI/README.md b/intel_extension_for_transformers/neural_chat/examples/serving/TGI/README.md
index 74b4ecd80ac..03c325a6652 100644
--- a/intel_extension_for_transformers/neural_chat/examples/serving/TGI/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/serving/TGI/README.md
@@ -89,4 +89,4 @@ curl ${your_ip}:${your_port}/v1/tgi/generate \
-H 'Content-Type: application/json'
```
-Of course, you can also consume the service via `postman`, `http request`, or other ways.
\ No newline at end of file
+Of course, you can also consume the service via `postman`, `http request`, or other ways.
diff --git a/intel_extension_for_transformers/neural_chat/examples/serving/vllm/README.md b/intel_extension_for_transformers/neural_chat/examples/serving/vllm/README.md
index 729b873c5b8..78d99a45319 100644
--- a/intel_extension_for_transformers/neural_chat/examples/serving/vllm/README.md
+++ b/intel_extension_for_transformers/neural_chat/examples/serving/vllm/README.md
@@ -21,4 +21,4 @@ neuralchat_server start --config_file textbot_vllm.yaml
```
curl -X POST -H "Content-Type: application/json" -d '{"prompt": "Tell me about Intel Xeon processors."}' http://localhost:8000/v1/chat/completions
-```
\ No newline at end of file
+```
diff --git a/intel_extension_for_transformers/neural_chat/examples/serving/vllm/textbot_vllm.yaml b/intel_extension_for_transformers/neural_chat/examples/serving/vllm/textbot_vllm.yaml
index 3e78f742930..91de0d5dbf7 100644
--- a/intel_extension_for_transformers/neural_chat/examples/serving/vllm/textbot_vllm.yaml
+++ b/intel_extension_for_transformers/neural_chat/examples/serving/vllm/textbot_vllm.yaml
@@ -41,4 +41,4 @@ serving:
max_context_len_to_capture: 8192
# task choices = ['textchat', 'voicechat', 'retrieval', 'text2image', 'finetune']
-tasks_list: ['textchat']
\ No newline at end of file
+tasks_list: ['textchat']
diff --git a/intel_extension_for_transformers/neural_chat/models/__init__.py b/intel_extension_for_transformers/neural_chat/models/__init__.py
index 407b29e53be..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/models/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/models/__init__.py
@@ -14,4 +14,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
diff --git a/intel_extension_for_transformers/neural_chat/models/model_utils.py b/intel_extension_for_transformers/neural_chat/models/model_utils.py
index c8db4e738c4..43ca611faac 100644
--- a/intel_extension_for_transformers/neural_chat/models/model_utils.py
+++ b/intel_extension_for_transformers/neural_chat/models/model_utils.py
@@ -961,7 +961,7 @@ def predict_stream(**params):
`num_return_sequences` (int): Specifies the number of alternative sequences to generate.
`bad_words_ids` (list or None): Contains a list of token IDs that should not appear in the generated text.
`force_words_ids` (list or None): Contains a list of token IDs that must be included in the generated text.
- `use_hpu_graphs` (bool):
+ `use_hpu_graphs` (bool):
Determines whether to utilize Habana Processing Units (HPUs) for accelerated generation.
`use_cache` (bool): Determines whether to utilize kv cache for accelerated generation.
`ipex_int8` (bool): Whether to use IPEX int8 model to inference.
@@ -1036,7 +1036,7 @@ def predict_stream(**params):
return
generate_kwargs = get_generate_kwargs(
- max_new_tokens, input_token_len,
+ max_new_tokens, input_token_len,
get_stop_token_ids(model, tokenizer),
assistant_model=assistant_model
)
@@ -1256,7 +1256,7 @@ def predict(**params):
`num_return_sequences` (int): Specifies the number of alternative sequences to generate.
`bad_words_ids` (list or None): Contains a list of token IDs that should not appear in the generated text.
`force_words_ids` (list or None): Contains a list of token IDs that must be included in the generated text.
- `use_hpu_graphs` (bool):
+ `use_hpu_graphs` (bool):
Determines whether to utilize Habana Processing Units (HPUs) for accelerated generation.
`use_cache` (bool): Determines whether to utilize kv cache for accelerated generation.
`ipex_int8` (bool): Whether to use IPEX int8 model to inference.
@@ -1321,8 +1321,8 @@ def predict(**params):
input_tokens, input_token_len = tokenization(prompt, tokenizer, device)
generate_kwargs = get_generate_kwargs(
- max_new_tokens, input_token_len,
- get_stop_token_ids(model, tokenizer),
+ max_new_tokens, input_token_len,
+ get_stop_token_ids(model, tokenizer),
assistant_model=assistant_model
)
diff --git a/intel_extension_for_transformers/neural_chat/models/solar_model.py b/intel_extension_for_transformers/neural_chat/models/solar_model.py
index d1f29ae1028..42d54d698f7 100644
--- a/intel_extension_for_transformers/neural_chat/models/solar_model.py
+++ b/intel_extension_for_transformers/neural_chat/models/solar_model.py
@@ -66,4 +66,3 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
return get_conv_template("solar")
register_model_adapter(SolarModel)
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/__init__.py
index 74a982ca49c..7df490129b8 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/__init__.py
@@ -16,4 +16,3 @@
# limitations under the License.
"""Intel® Neural Chat: An open-source Python library supporting LLM chatbot."""
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/finetuning/tts_finetuning.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/finetuning/tts_finetuning.py
index ebcdbfd83b1..7c108518c66 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/finetuning/tts_finetuning.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/finetuning/tts_finetuning.py
@@ -72,7 +72,7 @@ def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) ->
batch["speaker_embeddings"] = torch.tensor(speaker_features)
return batch
-
+
class TTSFinetuning:
def __init__(self, finetuning_config: TTSFinetuningConfig):
self.dataset_args, self.model_args = (
@@ -108,7 +108,7 @@ def _construct_audio_list(self):
audio_names = os.listdir(self.audio_folder_path)
audio_paths = [os.path.join(self.audio_folder_path, i) for i in audio_names]
return audio_paths
-
+
def _construct_finetuning_dataset(self):
raw_texts, normalized_texts = self._construct_text_list()
audio_paths = self._construct_audio_list()
@@ -203,4 +203,4 @@ def finetune(self):
)
trainer.train()
- return self.model
\ No newline at end of file
+ return self.model
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/requirements.txt b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/requirements.txt
index 1cfd62866c2..b7de6e9bdd6 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/requirements.txt
@@ -1,11 +1,11 @@
+datasets
+librosa
+num2words
paddlepaddle
paddlespeech
-transformers
-soundfile
-datasets
pydub
python-multipart
+soundfile
speechbrain
-librosa
+transformers
zhconv
-num2words
\ No newline at end of file
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/cut_video.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/cut_video.py
index 79a9d4c152a..b659d09052e 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/cut_video.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/cut_video.py
@@ -54,7 +54,7 @@ def cut_video(args, outdir):
end_min = 0
end_sec = 0
mark = 0
-
+
for i in range(0, min + 1, delta_X):
logging.info("i: %s", i)
if min >= delta_X:
@@ -72,7 +72,7 @@ def cut_video(args, outdir):
end_min = t_min
if i == t_min:
- end_sec = t_second
+ end_sec = t_second
start_hour = '0' + str(start_hour) if len(str(start_hour)) == 1 else str(start_hour)
start_min = '0' + str(start_min) if len(str(start_min)) == 1 else str(start_min)
@@ -96,7 +96,7 @@ def cut_video(args, outdir):
start_hour = int(end_hour)
start_min = int(end_min)
start_sec = int(end_sec)
-
+
if __name__ == '__main__':
parser = argparse.ArgumentParser(__doc__)
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/english_normalizer.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/english_normalizer.py
index 9b4e390b4ed..772425142df 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/english_normalizer.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/english_normalizer.py
@@ -54,7 +54,7 @@ def __init__(self):
"Z": "zed",
".": "point",
}
-
+
def correct_abbreviation(self, text):
# TODO mixed abbreviation or proper noun like i7, ffmpeg, BTW should be supported
@@ -120,4 +120,4 @@ def correct_number(self, text):
# if the text is not truncated correctly by early stop token, then manually add one.
if len(results) > 0 and results[-1] not in [",", ".", "?", "!"]:
results += "."
- return results
\ No newline at end of file
+ return results
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/reduce_noise.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/reduce_noise.py
index 9f3c1c63c08..fd5aeac2b07 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/reduce_noise.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/reduce_noise.py
@@ -370,4 +370,3 @@ def reduce_audio_amplify(self, output_audio_path, y):
rn_am_sound = rn_sound.apply_gain(original_db - rn_db)
rn_am_sound.export(f"{output_audio_path}_rn_ap.wav")
return f"{output_audio_path}_rn_ap.wav"
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/split_audio.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/split_audio.py
index be46fc15dfc..4f46ce7f7d5 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/split_audio.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/split_audio.py
@@ -164,7 +164,7 @@ def vad_collector(sample_rate, frame_duration_ms,
def main(args):
is_dir = os.path.isdir(args.in_path)
- if is_dir:
+ if is_dir:
input_dir = args.in_path
path_list=os.listdir(args.in_path)
else: # input is a wav file
@@ -202,4 +202,4 @@ def main(args):
if not is_exist:
logging.info("path not existed!")
else:
- main(args)
\ No newline at end of file
+ main(args)
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/video_to_wav.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/video_to_wav.py
index db91a5406d7..782ecfd223f 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/video_to_wav.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/utils/video_to_wav.py
@@ -66,7 +66,7 @@ def convert_video_to_wav(path, output_sample_rate, is_mono=True):
else: # pragma: no cover
logging.info("file %s format not supported!", filename)
continue
-
+
if __name__ == '__main__': # pragma: no cover
parser = argparse.ArgumentParser(__doc__)
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/cache_config.yaml b/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/cache_config.yaml
index c04a71fd905..f2995339340 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/cache_config.yaml
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/cache_config.yaml
@@ -33,7 +33,7 @@ storage_config:
sqlite,faiss
vector_params:
# Set vector storage related params here
-evaluation:
+evaluation:
distance
evaluation_config:
max_distance: 1.0
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/requirements.txt b/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/requirements.txt
index f92d2459735..f52106d5367 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/caching/requirements.txt
@@ -1,2 +1,2 @@
-gptcache
git+https://github.com/UKPLab/sentence-transformers.git@5c838a705c24c2dfd151a71674c99d09d014c1a9
+gptcache
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/diffusion_utils.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/diffusion_utils.py
index b20b83038d3..811b27ce1bf 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/diffusion_utils.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/diffusion_utils.py
@@ -37,4 +37,3 @@ def neural_engine_init(ir_path):
vae_decoder_graph.graph_init(vae_decoder_conf, vae_decoder_bin)
return [text_encoder_graph, unet_graph, vae_decoder_graph]
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/instructpix2pix_pipeline.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/instructpix2pix_pipeline.py
index 760c0ad3065..73e0dd95e09 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/instructpix2pix_pipeline.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/image2image/instructpix2pix_pipeline.py
@@ -341,7 +341,7 @@ def __call__(
# Original Pytorch Diffuser Unet Code: predict the noise residual
#noise_pred = self.unet(scaled_latent_model_input, t, encoder_hidden_states=prompt_embeds).sample
-
+
# The ITREX Unet Code
scaled_latent_model_input = scaled_latent_model_input.contiguous()
prompt_embeds = prompt_embeds.contiguous()
@@ -744,4 +744,3 @@ def bf16_to_fp32(bf16_np):
int32_np = int32_np << 16
fp32_np = int32_np.view(np.float32)
return fp32_np
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/memory/memory.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/memory/memory.py
index f19147517bf..61b8388a937 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/memory/memory.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/memory/memory.py
@@ -20,7 +20,7 @@
class Memory:
def __init__(self):
self.chat_memory = []
-
+
def clear(self):
self.chat_memory.clear()
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/README.md b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/README.md
index ba961cf5de0..2a5dbe5f8ff 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/README.md
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/README.md
@@ -98,4 +98,4 @@ threads [int]: The thread number of model inference. Set to the core number of y
max_new_tokens [int]: The max generated token numbers. Default to 32.
seed [int]: The random seed of llm. Default to 1234.
-```
\ No newline at end of file
+```
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py
index ad599796375..83f1f944eed 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py
@@ -55,4 +55,3 @@ def ner_inference(self, response):
logging.info("[NER info] Inference time consumption: %s", time.time() - start_time)
return result
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/requirements.txt b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/requirements.txt
index b642f9a3f58..04e761b52bd 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/requirements.txt
@@ -1,10 +1,10 @@
+deepface
+exifread
+intel_extension_for_pytorch
+neural-compressor
+pymysql
+spacy
torch
-torchvision
torchaudio
-spacy
-neural-compressor
+torchvision
transformers
-intel_extension_for_pytorch
-pymysql
-deepface
-exifread
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/utils/process_text.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/utils/process_text.py
index 2ed551c4864..b9f82a1fa5f 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/utils/process_text.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/utils/process_text.py
@@ -57,7 +57,7 @@ def process_time(result_text: str, doc) -> dict:
if len(mentioned_time["period"]) % 2 != 0:
mentioned_time["time"] = list(set(mentioned_time["time"]+mentioned_time["period"]))
mentioned_time["period"] = []
-
+
return mentioned_time
@@ -94,11 +94,11 @@ def process_entities(query, doc, mentioned_time: dict) -> dict:
result_period = post_process_last_week()
result = {
- "period": result_period,
- "time": mentioned_time['time'],
- 'location': location,
- "name": name,
+ "period": result_period,
+ "time": mentioned_time['time'],
+ 'location': location,
+ "name": name,
"organization": organization
}
-
- return result
\ No newline at end of file
+
+ return result
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/prompt/prompt_template.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/prompt/prompt_template.py
index 14a8f24adc5..ec40cb093da 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/prompt/prompt_template.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/prompt/prompt_template.py
@@ -69,4 +69,4 @@ def generate_intent_prompt(query):
conv = PromptTemplate("intent")
conv.append_message(conv.roles[0], query)
conv.append_message(conv.roles[1], None)
- return conv.get_prompt()
\ No newline at end of file
+ return conv.get_prompt()
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/README.md b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/README.md
index bde22c5916a..bcf0291fb31 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/README.md
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/README.md
@@ -236,4 +236,4 @@ from intel_extension_for_transformers.langchain.retrievers import ChildParentRet
from langchain.vectorstores import Chroma
retriever = ChildParentRetriever(vectorstore=Chroma(documents=child_documents), parentstore=Chroma(documents=parent_documents), search_type=xxx, search_kwargs={...})
docs=retriever.get_relevant_documents("Intel")
-```
\ No newline at end of file
+```
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/detector/intent_detection.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/detector/intent_detection.py
index 8aa265e0476..895011d3e86 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/detector/intent_detection.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/detector/intent_detection.py
@@ -36,4 +36,3 @@ def intent_detection(self, model_name, query):
params["max_new_tokens"] = 10
intent = predict(**params)
return intent
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/html_parser.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/html_parser.py
index 6c6fa42e18e..3b6aa70f382 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/html_parser.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/html_parser.py
@@ -193,4 +193,4 @@ def load_html_data(url):
# {'text': all_text, 'main_content': main_content}
- return main_content
\ No newline at end of file
+ return main_content
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/parser.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/parser.py
index 1d879466ae6..3337470f430 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/parser.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/parser/parser.py
@@ -37,8 +37,8 @@ def __init__(self, max_chuck_size=512, min_chuck_size=5, process=True):
self.max_chuck_size = max_chuck_size
self.min_chuck_size = min_chuck_size
self.process = process
-
-
+
+
def load(self, input, **kwargs):
"""
The API for loading the file. Support single file, batch files, and urls parsing.
@@ -49,7 +49,7 @@ def load(self, input, **kwargs):
self.min_chuck_size = kwargs['min_chuck_size']
if 'process' in kwargs:
self.process = kwargs['process']
-
+
if isinstance(input, str):
if os.path.isfile(input):
data_collection = self.parse_document(input)
@@ -64,10 +64,10 @@ def load(self, input, **kwargs):
logging.error("The given link/str is unavailable. Please try another one!")
else:
logging.error("The input format is invalid!")
-
+
return data_collection
-
-
+
+
def parse_document(self, input):
"""
Parse the uploaded file.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retrieval_agent.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retrieval_agent.py
index 4aaa8c75490..87c116867e7 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retrieval_agent.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retrieval_agent.py
@@ -90,12 +90,12 @@ def __init__(self,
"accuracy",
"general",
)
-
+
assert self.retrieval_type in allowed_retrieval_type, "search_type of {} not allowed.".format( \
self.retrieval_type)
assert self.mode in allowed_generation_mode, "generation mode of {} not allowed.".format( \
self.mode)
-
+
if isinstance(input_path, str):
if os.path.exists(input_path):
self.input_path = input_path
@@ -129,12 +129,12 @@ def __init__(self,
except Exception as e:
logging.error("Please select a proper embedding model.")
logging.error(e)
-
+
self.document_parser = DocumentParser(max_chuck_size=max_chuck_size, min_chuck_size = min_chuck_size, \
process=self.process)
data_collection = self.document_parser.load(input=self.input_path, **kwargs)
logging.info("The parsing for the uploaded files is finished.")
-
+
langchain_documents = document_transfer(data_collection)
logging.info("The format of parsed documents is transferred.")
@@ -203,7 +203,7 @@ def create(self, input_path, **kwargs):
"""
data_collection = self.document_parser.load(input=input_path, **kwargs)
langchain_documents = document_transfer(data_collection)
-
+
if self.retrieval_type == 'default':
knowledge_base = self.database.from_documents(documents=langchain_documents, \
embedding=self.embeddings, **kwargs)
@@ -219,14 +219,14 @@ def create(self, input_path, **kwargs):
self.retriever = RetrieverAdapter(retrieval_type=self.retrieval_type, document_store=knowledge_base, \
child_document_store=child_knowledge_base, **kwargs).retriever
logging.info("The retriever is successfully built.")
-
+
def append_localdb(self, append_path, **kwargs):
"Append the knowledge instances into a given knowledge base."
data_collection = self.document_parser.load(input=append_path, **kwargs)
langchain_documents = document_transfer(data_collection)
-
+
if self.retrieval_type == 'default':
knowledge_base = self.database.from_documents(documents=langchain_documents, \
embedding=self.embeddings, **kwargs)
@@ -242,7 +242,7 @@ def append_localdb(self, append_path, **kwargs):
self.retriever = RetrieverAdapter(retrieval_type=self.retrieval_type, document_store=knowledge_base, \
child_document_store=child_knowledge_base, **kwargs).retriever
logging.info("The retriever is successfully built.")
-
+
def pre_llm_inference_actions(self, model_name, query):
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retriever_adapter.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retriever_adapter.py
index 5aec1b1ffee..27582b9627a 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retriever_adapter.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/retrieval/retriever_adapter.py
@@ -30,7 +30,7 @@ class RetrieverAdapter():
def __init__(self, retrieval_type='default', document_store=None, child_document_store= None, **kwargs):
self.retrieval_type = retrieval_type
-
+
if self.retrieval_type == "default":
self.retriever = VectorStoreRetriever(vectorstore = document_store, **kwargs)
elif self.retrieval_type == "child_parent":
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/dict.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/dict.py
index 7c0e9851087..b205c0b0168 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/dict.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/dict.py
@@ -1938,4 +1938,4 @@
['政治', '司法'],
['政治', '主权'],
['政治', '北京大桥']
-]
\ No newline at end of file
+]
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/stopword.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/stopword.py
index dcfc0d51ce1..9aeb3533bc1 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/stopword.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/security/stopword.py
@@ -63,4 +63,4 @@
'……',
'《',
'》',
-]
\ No newline at end of file
+]
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/README.md b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/README.md
index 33e710c6afe..ce4b2d72d0f 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/README.md
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/README.md
@@ -27,4 +27,4 @@ sadtalker.convert(source_image=source_image, driven_audio=driven_audio, output_v
## Acknowledgements
-This plugin is mostly adapted from [SadTalker](https://github.com/OpenTalker/SadTalker). We thank the related authors for their great work!
\ No newline at end of file
+This plugin is mostly adapted from [SadTalker](https://github.com/OpenTalker/SadTalker). We thank the related authors for their great work!
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/inference.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/inference.py
index f0e045769b5..1c2bc280417 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/inference.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/inference.py
@@ -57,7 +57,7 @@ def main(args):
batch_size = args.batch_size
current_root_path = os.path.split(sys.argv[0])[0]
-
+
sadtalker_paths = init_path(
args.checkpoint_dir, os.path.join(current_root_path, "src/config"), args.size, args.preprocess
)
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/requirements.txt b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/requirements.txt
index cb3c4622e26..b59f6263562 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/requirements.txt
@@ -1,21 +1,21 @@
-numpy==1.23.5
+av
+basicsr==1.4.2
+dlib-bin
face_alignment==1.3.5
+facexlib @ git+https://github.com/Spycsh/facexlib@master
+gfpgan
imageio==2.19.3
imageio-ffmpeg==0.4.7
+joblib==1.2.0
+kornia==0.6.8
librosa==0.9.2 #
numba
-resampy==0.3.1
+numpy==1.23.5
pydub==0.25.1
+pyyaml
+resampy==0.3.1
+safetensors
+scikit-image==0.19.3
scipy==1.10.1
-kornia==0.6.8
tqdm
yacs==0.1.8
-pyyaml
-joblib==1.2.0
-scikit-image==0.19.3
-basicsr==1.4.2
-facexlib @ git+https://github.com/Spycsh/facexlib@master
-gfpgan
-dlib-bin
-av
-safetensors
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2exp_models/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2exp_models/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2exp_models/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2exp_models/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2pose_models/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2pose_models/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2pose_models/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/audio2pose_models/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2exp.yaml b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2exp.yaml
index 3640ea4a496..1691e739f76 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2exp.yaml
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2exp.yaml
@@ -27,7 +27,7 @@ DATASET:
DEBUG: True
NUM_REPEATS: 2
T: 40
-
+
MODEL:
FRAMEWORK: V2
@@ -44,7 +44,7 @@ MODEL:
LATENT_SIZE: 256
ENCODER_LAYER_SIZES: [192, 1024]
DECODER_LAYER_SIZES: [1024, 192]
-
+
TRAIN:
MAX_EPOCH: 300
@@ -67,5 +67,3 @@ TRAIN:
TAG:
NAME: small_dataset
-
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2pose.yaml b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2pose.yaml
index 12335c7baa5..002af7c09d1 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2pose.yaml
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/auido2pose.yaml
@@ -25,7 +25,7 @@ DATASET:
COEFF_LEN: 73
NUM_CLASSES: 46
DEBUG: True
-
+
MODEL:
AUDIOENCODER:
@@ -41,7 +41,7 @@ MODEL:
LATENT_SIZE: 64
ENCODER_LAYER_SIZES: [192, 128]
DECODER_LAYER_SIZES: [128, 192]
-
+
TRAIN:
MAX_EPOCH: 150
@@ -59,5 +59,3 @@ TRAIN:
TAG:
NAME: cvae_UNET_useAudio_usewav2lipAudioEncoder
-
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender.yaml b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender.yaml
index d8d7949ad1f..e827ce9fcb0 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender.yaml
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender.yaml
@@ -18,20 +18,20 @@
model_params:
common_params:
- num_kp: 15
- image_channel: 3
+ num_kp: 15
+ image_channel: 3
feature_channel: 32
estimate_jacobian: False # True
kp_detector_params:
temperature: 0.1
- block_expansion: 32
+ block_expansion: 32
max_features: 1024
scale_factor: 0.25 # 0.25
num_blocks: 5
reshape_channel: 16384 # 16384 = 1024 * 16
reshape_depth: 16
he_estimator_params:
- block_expansion: 64
+ block_expansion: 64
max_features: 2048
num_bins: 66
generator_params:
@@ -50,7 +50,7 @@ model_params:
compress: 4
discriminator_params:
scales: [1]
- block_expansion: 32
+ block_expansion: 32
max_features: 512
num_blocks: 4
sn: True
@@ -60,4 +60,3 @@ model_params:
layer: 3
num_kp: 15
num_bins: 66
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender_still.yaml b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender_still.yaml
index e6cd9a48a89..ac2889f2318 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender_still.yaml
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/config/facerender_still.yaml
@@ -18,20 +18,20 @@
model_params:
common_params:
- num_kp: 15
- image_channel: 3
+ num_kp: 15
+ image_channel: 3
feature_channel: 32
estimate_jacobian: False # True
kp_detector_params:
temperature: 0.1
- block_expansion: 32
+ block_expansion: 32
max_features: 1024
scale_factor: 0.25 # 0.25
num_blocks: 5
reshape_channel: 16384 # 16384 = 1024 * 16
reshape_depth: 16
he_estimator_params:
- block_expansion: 64
+ block_expansion: 64
max_features: 2048
num_bins: 66
generator_params:
@@ -50,7 +50,7 @@ model_params:
compress: 4
discriminator_params:
scales: [1]
- block_expansion: 32
+ block_expansion: 32
max_features: 512
num_blocks: 4
sn: True
@@ -60,4 +60,3 @@ model_params:
layer: 3
num_kp: 15
num_bins: 66
-
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/face3d/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/face3d/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/face3d/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/face3d/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/make_animation.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/make_animation.py
index aebffd83ddd..2a63e08eb7c 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/make_animation.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/facerender/modules/make_animation.py
@@ -261,4 +261,4 @@ def natural_keys(text):
# predictions_ts = torch.stack(predictions, dim=1)
predictions_ts = torch.stack(aggregated_predictions, dim=1)
- return predictions_ts
\ No newline at end of file
+ return predictions_ts
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/__init__.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/__init__.py
index 2823243c0bb..18896e7b549 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/__init__.py
@@ -13,4 +13,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/audio.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/audio.py
index c43b74a8b0c..7af1e6fa07f 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/audio.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/audio.py
@@ -96,4 +96,4 @@ def _normalize(S):
if hp.symmetric_mels:
return (2 * hp.max_abs_value) * ((S - hp.min_level_db) / (-hp.min_level_db)) - hp.max_abs_value
else:
- return hp.max_abs_value * ((S - hp.min_level_db) / (-hp.min_level_db))
\ No newline at end of file
+ return hp.max_abs_value * ((S - hp.min_level_db) / (-hp.min_level_db))
diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/videoio.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/videoio.py
index bbcba1e1f5a..b554d63a8b3 100644
--- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/videoio.py
+++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/video/face_animation/src/utils/videoio.py
@@ -43,4 +43,4 @@ def save_video_with_watermark(video, audio, save_path, watermark=False):
input_audio = ffmpeg.input(audio)
ffmpeg.concat(input_video, input_audio, v=1, a=1).output(
save_path
- ).run()
\ No newline at end of file
+ ).run()
diff --git a/intel_extension_for_transformers/neural_chat/plugins.py b/intel_extension_for_transformers/neural_chat/plugins.py
index 859b5cff380..624c4351212 100644
--- a/intel_extension_for_transformers/neural_chat/plugins.py
+++ b/intel_extension_for_transformers/neural_chat/plugins.py
@@ -72,4 +72,4 @@ def get_registered_plugins():
def get_all_plugins():
return ["tts", "tts_chinese", "asr", "asr_chinese", "retrieval", "cache", "safety_checker", "ner", "ner_int",
- "face_animation"]
\ No newline at end of file
+ "face_animation"]
diff --git a/intel_extension_for_transformers/neural_chat/prompts/__init__.py b/intel_extension_for_transformers/neural_chat/prompts/__init__.py
index 9d709dd806c..d3e17f82b3e 100644
--- a/intel_extension_for_transformers/neural_chat/prompts/__init__.py
+++ b/intel_extension_for_transformers/neural_chat/prompts/__init__.py
@@ -15,4 +15,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .prompt import PromptTemplate
\ No newline at end of file
+from .prompt import PromptTemplate
diff --git a/intel_extension_for_transformers/neural_chat/prompts/prompt.py b/intel_extension_for_transformers/neural_chat/prompts/prompt.py
index 6627205a2a0..2686417bd93 100644
--- a/intel_extension_for_transformers/neural_chat/prompts/prompt.py
+++ b/intel_extension_for_transformers/neural_chat/prompts/prompt.py
@@ -196,9 +196,9 @@
register_conv_template(
Conversation(
name="ner",
- system_message="""Please determine the precise time mentioned in the user's query.
- Your response should consist only of an accurate time in the format
- 'Time: YYYY-MM-DD' or 'Period: YYYY-MM-DD to YYYY-MM-DD.'
+ system_message="""Please determine the precise time mentioned in the user's query.
+ Your response should consist only of an accurate time in the format
+ 'Time: YYYY-MM-DD' or 'Period: YYYY-MM-DD to YYYY-MM-DD.'
If the user query does not include any time reference, please reply with 'None'.\n""",
roles=("Current Time: ", "User Query: "),
sep_style=SeparatorStyle.NO_COLON_SINGLE,
diff --git a/intel_extension_for_transformers/neural_chat/requirements.txt b/intel_extension_for_transformers/neural_chat/requirements.txt
index ec4654403e6..27ccc939601 100644
--- a/intel_extension_for_transformers/neural_chat/requirements.txt
+++ b/intel_extension_for_transformers/neural_chat/requirements.txt
@@ -1,29 +1,29 @@
-transformers>=4.35.2
-peft==0.6.2
+accelerate
+cchardet
+einops
+evaluate
+fastapi==0.103.2
fschat==0.2.32
-torch==2.1.0
-torchaudio==2.1.0
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
+huggingface_hub
intel_extension_for_pytorch
+neural-compressor
+neural_speed
+numpy==1.23.5
onnx>=1.15.0
-shortuuid
-evaluate
-fastapi==0.103.2
-pydantic==1.10.13
-starlette
-yacs
-uvicorn
optimum
optimum-intel
+peft==0.6.2
+pydantic==1.10.13
+python-dotenv
+python-multipart
rouge_score
-numpy==1.23.5
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
-accelerate
-cchardet
+shortuuid
+starlette
tensorflow>=2.13.0
-neural-compressor
-einops
-python-multipart
-python-dotenv
-huggingface_hub
+torch==2.1.0
+torchaudio==2.1.0
+transformers>=4.35.2
+uvicorn
vllm
-neural_speed
+yacs
diff --git a/intel_extension_for_transformers/neural_chat/requirements_cpu.txt b/intel_extension_for_transformers/neural_chat/requirements_cpu.txt
index 3e741d24572..48a5329c47a 100644
--- a/intel_extension_for_transformers/neural_chat/requirements_cpu.txt
+++ b/intel_extension_for_transformers/neural_chat/requirements_cpu.txt
@@ -1,26 +1,26 @@
-transformers>=4.35.2
-peft==0.6.2
-fschat==0.2.32
-intel_extension_for_pytorch==2.1.0
-shortuuid
+--extra-index-url https://download.pytorch.org/whl/cpu
+cchardet
+einops
evaluate
fastapi==0.103.2
-pydantic==1.10.13
-starlette
-yacs
-uvicorn
+fschat==0.2.32
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
+intel_extension_for_pytorch==2.1.0
+neural-compressor
+neural_speed
+numpy==1.23.5
optimum
optimum-intel
+peft==0.6.2
+pydantic==1.10.13
+python-dotenv
+python-multipart
rouge_score
-numpy==1.23.5
+shortuuid
+starlette
tiktoken==0.4.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
---extra-index-url https://download.pytorch.org/whl/cpu
torch==2.1.0
torchaudio==2.1.0
-neural-compressor
-einops
-cchardet
-python-multipart
-python-dotenv
-neural_speed
+transformers>=4.35.2
+uvicorn
+yacs
diff --git a/intel_extension_for_transformers/neural_chat/requirements_hpu.txt b/intel_extension_for_transformers/neural_chat/requirements_hpu.txt
index 5f1a632fb4f..6700cfa2c87 100644
--- a/intel_extension_for_transformers/neural_chat/requirements_hpu.txt
+++ b/intel_extension_for_transformers/neural_chat/requirements_hpu.txt
@@ -1,20 +1,20 @@
-transformers>=4.35.2
-peft
-fschat==0.2.32
-shortuuid
+cchardet
+einops
evaluate
fastapi==0.103.2
-pydantic==1.10.13
-starlette
-yacs
-uvicorn
-optimum
-rouge_score
-numpy==1.23.5
+fschat==0.2.32
git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
-neural-compressor
intel_extension_for_pytorch
-einops
-cchardet
-python-multipart
+neural-compressor
+numpy==1.23.5
+optimum
+peft
+pydantic==1.10.13
python-dotenv
+python-multipart
+rouge_score
+shortuuid
+starlette
+transformers>=4.35.2
+uvicorn
+yacs
diff --git a/intel_extension_for_transformers/neural_chat/requirements_pc.txt b/intel_extension_for_transformers/neural_chat/requirements_pc.txt
index 5a5aa9001cf..84edc0c4f70 100644
--- a/intel_extension_for_transformers/neural_chat/requirements_pc.txt
+++ b/intel_extension_for_transformers/neural_chat/requirements_pc.txt
@@ -1,23 +1,23 @@
-transformers
-peft
-fschat==0.2.32
-shortuuid
+--extra-index-url https://download.pytorch.org/whl/cpu
+cchardet
+einops
evaluate
fastapi==0.103.2
-pydantic==1.10.13
-starlette
-yacs
-uvicorn
+fschat==0.2.32
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
+neural-compressor
+numpy==1.23.5
optimum
optimum-intel
+peft
+pydantic==1.10.13
+python-dotenv
+python-multipart
rouge_score
-numpy==1.23.5
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
---extra-index-url https://download.pytorch.org/whl/cpu
+shortuuid
+starlette
torch==2.1.0
torchaudio==2.1.0
-neural-compressor
-einops
-cchardet
-python-multipart
-python-dotenv
+transformers
+uvicorn
+yacs
diff --git a/intel_extension_for_transformers/neural_chat/requirements_xpu.txt b/intel_extension_for_transformers/neural_chat/requirements_xpu.txt
index c3d30f8ffca..f2a980c0e9c 100644
--- a/intel_extension_for_transformers/neural_chat/requirements_xpu.txt
+++ b/intel_extension_for_transformers/neural_chat/requirements_xpu.txt
@@ -1,16 +1,16 @@
-transformers>=4.35.2
-fschat==0.2.32
-shortuuid
+cchardet
+einops
evaluate
fastapi==0.103.2
+fschat==0.2.32
+neural-compressor
+numpy==1.23.5
pydantic==1.10.13
+python-dotenv
+python-multipart
+rouge_score
+shortuuid
starlette
-yacs
+transformers>=4.35.2
uvicorn
-rouge_score
-numpy==1.23.5
-cchardet
-neural-compressor
-einops
-python-multipart
-python-dotenv
+yacs
diff --git a/intel_extension_for_transformers/neural_chat/server/restful/codegen_api.py b/intel_extension_for_transformers/neural_chat/server/restful/codegen_api.py
index 22be5bc444c..d000af762ad 100644
--- a/intel_extension_for_transformers/neural_chat/server/restful/codegen_api.py
+++ b/intel_extension_for_transformers/neural_chat/server/restful/codegen_api.py
@@ -33,7 +33,7 @@ def check_completion_request(request: BaseModel) -> Optional[str]:
logger.info(f"Checking parameters of completion request...")
if request.temperature is not None and request.temperature < 0:
return f"Param Error: {request.temperature} is less than the minimum of 0 --- 'temperature'"
-
+
if request.temperature is not None and request.temperature > 2:
return f"Param Error: {request.temperature} is greater than the maximum of 2 --- 'temperature'"
diff --git a/intel_extension_for_transformers/neural_chat/server/restful/faceanimation_api.py b/intel_extension_for_transformers/neural_chat/server/restful/faceanimation_api.py
index 8c31c8d8162..e325236d112 100644
--- a/intel_extension_for_transformers/neural_chat/server/restful/faceanimation_api.py
+++ b/intel_extension_for_transformers/neural_chat/server/restful/faceanimation_api.py
@@ -102,4 +102,4 @@ async def handle_talkingbot_face_animation(image: UploadFile = File(...),
text=text,
mode=mode,
voice=voice)
- return response
\ No newline at end of file
+ return response
diff --git a/intel_extension_for_transformers/neural_chat/server/restful/finetune_api.py b/intel_extension_for_transformers/neural_chat/server/restful/finetune_api.py
index facb7a2c986..749d00c9170 100644
--- a/intel_extension_for_transformers/neural_chat/server/restful/finetune_api.py
+++ b/intel_extension_for_transformers/neural_chat/server/restful/finetune_api.py
@@ -36,7 +36,7 @@ def check_finetune_request(request: BaseModel) -> Optional[str]:
if request.train_file is None and request.dataset_name is None:
return f"Param Error: finetune dataset can not be None"
return None
-
+
class FinetuneAPIRouter(APIRouter):
@@ -56,7 +56,7 @@ def get_chatbot(self):
logger.error("Chatbot instance is not found.")
raise RuntimeError("Chatbot instance has not been set.")
return self.chatbot
-
+
def handle_finetune_request(self, request: FinetuneRequest) -> str:
try:
model_args = ModelArguments(model_name_or_path=request.model_name_or_path)
@@ -90,6 +90,6 @@ def handle_finetune_request(self, request: FinetuneRequest) -> str:
@router.post("/v1/finetune")
async def finetune_endpoint(request: FinetuneRequest):
ret = check_finetune_request(request)
- if ret is not None:
+ if ret is not None:
raise RuntimeError(f"Invalid parameter: {ret}")
- return router.handle_finetune_request(request)
\ No newline at end of file
+ return router.handle_finetune_request(request)
diff --git a/intel_extension_for_transformers/neural_chat/server/restful/openai_protocol.py b/intel_extension_for_transformers/neural_chat/server/restful/openai_protocol.py
index 6c4883632ae..84f836bf698 100644
--- a/intel_extension_for_transformers/neural_chat/server/restful/openai_protocol.py
+++ b/intel_extension_for_transformers/neural_chat/server/restful/openai_protocol.py
@@ -16,7 +16,7 @@
# limitations under the License.
"""
-Code source from FastChat's OpenAI protocol:
+Code source from FastChat's OpenAI protocol:
https://github.com/lm-sys/FastChat/blob/main/fastchat/protocol/openai_api_protocol.py
"""
diff --git a/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py b/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py
index 08474bd3a32..e433d183850 100644
--- a/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py
+++ b/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py
@@ -59,7 +59,7 @@ def get_chatbot(self):
logger.error("Chatbot instance is not found.")
raise RuntimeError("Chatbot instance has not been set.")
return self.chatbot
-
+
async def handle_voice_chat_request(self, prompt: str, audio_output_path: Optional[str]=None) -> str:
chatbot = self.get_chatbot()
try:
@@ -120,7 +120,7 @@ async def handle_ai_photos_upload_images(request: Request, background_tasks: Bac
img_path = image_path+'/'+ img_name
# save exif info from origin image
exif = img_obj.info.get('exif', b"")
-
+
# save image info into db
empty_tags = '{}'
insert_sql = f"INSERT INTO image_info VALUES(null, '{user_id}', '{img_path}', null, '', \
@@ -164,7 +164,7 @@ def handle_ai_photos_get_all_images(request: Request):
result_list = []
mysql_db = MysqlDb()
image_list = mysql_db.fetch_all(
- sql=f'''SELECT image_id, image_path FROM image_info
+ sql=f'''SELECT image_id, image_path FROM image_info
WHERE user_id="{user_id}" AND exist_status="active";''')
for image in image_list:
image_name = image['image_path'].split('/')[-1]
@@ -250,22 +250,22 @@ async def handle_ai_photos_get_image_detail(request: Request):
try:
mysql_db = MysqlDb()
image_info = mysql_db.fetch_one(
- sql=f'''SELECT * FROM image_info WHERE
- image_id={image_id} AND user_id="{user_id}" AND exist_status="active";''',
+ sql=f'''SELECT * FROM image_info WHERE
+ image_id={image_id} AND user_id="{user_id}" AND exist_status="active";''',
params=None)
except Exception as e:
logger.error("Large Language Model
-
+
4th Gen Intel® Xeon® with Intel® AMX