Skip to content

Commit 59c6eda

Browse files
outbreak-senoutbreak-sen
authored
【开源实习】bit模型微调 (#1995)
1 parent 4adcdb8 commit 59c6eda

File tree

3 files changed

+272
-0
lines changed

3 files changed

+272
-0
lines changed

‎llm/finetune/bit/README.md

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# bit微调
2+
3+
实现了"HorcruxNo13/bit-50"模型在"dpdl-benchmark/oxford_flowers102"数据集上的微调实验。
4+
任务链接在https://gitee.com/mindspore/community/issues/IAUPCI
5+
transformers+pytorch+3090的benchmark是自己编写的，仓库位于https://github.com/outbreak-sen/Bit_flowers102_Finetune
6+
更改代码位于llm/finetune/bit，只包含mindnlp+mindspore的
7+
实验结果如下
8+
9+
## 硬件
10+
11+
资源规格：NPU: 1*Ascend-D910B(显存: 64GB), CPU: 24, 内存: 192GB
12+
13+
智算中心：武汉智算中心
14+
15+
镜像：mindspore_2_5_py311_cann8
16+
17+
torch训练硬件资源规格：Nvidia 3090
18+
19+
## 模型与数据集
20+
21+
模型："HorcruxNo13/bit-50"
22+
23+
数据集："dpdl-benchmark/oxford_flowers102"
24+
25+
## Eval Loss Values 表格
26+
27+
| Epoch | mindNLP | torch |
28+
|-------|---------------|---------------|
29+
| 1 | 3.5184175968 | 4.6460494995 |
30+
| 2 | 1.7758612633 | 4.2146801949 |
31+
| 3 | 0.9314232469 | 3.8055384159 |
32+
| 4 | 0.6095938683 | 3.4315345287 |
33+
| 5 | 0.4878421128 | 3.1143600941 |
34+
| 6 | 0.4401741028 | 2.8422958851 |
35+
| 7 | 0.4239776731 | 2.6192340851 |
36+
| 8 | 0.4162144363 | 2.4506986141 |
37+
| 9 | 0.4113974869 | 2.3450050354 |
38+
| 10 | 0.4095760584 | 2.2997686863 |
39+
40+
## Test Accuracy 表格
41+
42+
| Epoch | mindNLP | torch |
43+
|-------|---------------|---------------|
44+
| 1 | 0.9219 | 0.6225 |
45+
46+
## 图片分类测试
47+
48+
问题来自评估数据集的第一个问题，微调后结果准确
49+
50+
* 问题输入：
51+
dataset['test'][0]['image']
52+
* 真实标签：
53+
26
54+
* mindnlp未微调前的回答：
55+
25
56+
* mindnlp微调后的回答：
57+
26
58+
* torch微调前的回答：
59+
41
60+
* torch微调后的回答：
61+
26

‎llm/finetune/bit/mindNLP_Bit_flowers.py

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import mindspore as ms
2+
import mindspore.dataset as ds
3+
from datasets import load_dataset
4+
from mindnlp.transformers import (
5+
BitForImageClassification,
6+
AutoImageProcessor
7+
)
8+
from mindnlp.engine import Trainer, TrainingArguments
9+
import os
10+
import numpy as np
11+
ms.set_context(device_target="Ascend")
12+
model_name = "HorcruxNo13/bit-50"
13+
processor = AutoImageProcessor.from_pretrained(model_name)
14+
model = BitForImageClassification.from_pretrained(
15+
model_name,
16+
num_labels=102,
17+
ignore_mismatched_sizes=True
18+
)
19+
dataset = load_dataset("dpdl-benchmark/oxford_flowers102", split="train")
20+
# 将训练集按8:2的比例拆分为训练集和测试集
21+
dataset = dataset.train_test_split(test_size=0.2, seed=42)
22+
dataset.save_to_disk("./flowers102")
23+
24+
print(dataset)
25+
# 选择一个测试集样本进行测试
26+
test_image = dataset['test'][0]['image']
27+
test_label = dataset['test'][0]['label']
28+
29+
print("\n=== 训练参数 ===")
30+
training_args = TrainingArguments(
31+
output_dir="./mindNLP_bit_flowers102",
32+
evaluation_strategy="epoch",
33+
save_strategy="epoch",
34+
learning_rate=5e-5,
35+
per_device_train_batch_size=64,
36+
per_device_eval_batch_size=128,
37+
num_train_epochs=10,
38+
gradient_accumulation_steps=1,
39+
logging_steps=50,
40+
load_best_model_at_end=True,
41+
warmup_steps=0,
42+
weight_decay=0.01,
43+
remove_unused_columns=False,
44+
max_grad_norm=0.0 # 禁用梯度裁剪
45+
)
46+
print("\n=== 先生成np数据 ===")
47+
train_data = []
48+
train_labels = []
49+
for item in dataset['train']:
50+
img = item['image'].convert('RGB')
51+
inputs = processor(images=img, return_tensors="np", size={"height": 384, "width": 384})
52+
train_data.append(inputs['pixel_values'][0])
53+
train_labels.append(item['label'])
54+
test_data = []
55+
test_labels = []
56+
for item in dataset['test']:
57+
img = item['image'].convert('RGB')
58+
inputs = processor(images=img, return_tensors="np", size={"height": 384, "width": 384})
59+
test_data.append(inputs['pixel_values'][0])
60+
test_labels.append(item['label'])
61+
train_data = np.array(train_data, dtype=np.float32)
62+
train_labels = np.array(train_labels, dtype=np.int32)
63+
test_data = np.array(test_data, dtype=np.float32)
64+
test_labels = np.array(test_labels, dtype=np.int32)
65+
print("\n=== 将预处理后的数据集转换为MindSpore格式 ===")
66+
def create_mindspore_dataset(data, labels, batch_size, shuffle=True):
67+
dataset = ds.NumpySlicesDataset(
68+
{
69+
"pixel_values": data,
70+
"labels": labels
71+
},
72+
shuffle=shuffle
73+
)
74+
dataset = dataset.batch(batch_size, drop_remainder=True)
75+
return dataset
76+
77+
# 创建训练和评估数据集
78+
train_dataset = create_mindspore_dataset(
79+
train_data,
80+
train_labels,
81+
batch_size=training_args.per_device_train_batch_size,
82+
shuffle=True
83+
)
84+
85+
eval_dataset = create_mindspore_dataset(
86+
test_data,
87+
test_labels,
88+
batch_size=training_args.per_device_eval_batch_size,
89+
shuffle=False
90+
)
91+
92+
# 单图测试函数
93+
def test_single_image(model, processor, image):
94+
inputs = processor(
95+
images=image.convert('RGB'),
96+
return_tensors="ms",
97+
size={"height": 384, "width": 384}
98+
)
99+
model.set_train(False)
100+
outputs = model(**inputs)
101+
predictions = outputs.logits.argmax(-1)
102+
return predictions.asnumpy().item()
103+
104+
print("\n=== 训练前测试 ===")
105+
pred_before = test_single_image(model, processor, test_image)
106+
print(f"真实标签: {test_label}")
107+
print(f"预测标签: {pred_before}")
108+
109+
import evaluate
110+
import numpy as np
111+
from mindnlp.engine.utils import EvalPrediction
112+
113+
metric = evaluate.load("accuracy")
114+
# 添加调试信息
115+
def compute_metrics(eval_pred: EvalPrediction):
116+
logits, labels = eval_pred
117+
predictions = np.argmax(logits, axis=-1)
118+
result = metric.compute(predictions=predictions, references=labels)
119+
return result
120+
print("\n=== 创建Trainer实例 ===")
121+
trainer = Trainer(
122+
model=model,
123+
args=training_args,
124+
train_dataset=train_dataset,
125+
eval_dataset=eval_dataset,
126+
compute_metrics=compute_metrics,
127+
)
128+
# trainer = Trainer(
129+
# model=model,
130+
# args=training_args,
131+
# train_dataset=train_dataset,
132+
# eval_dataset=eval_dataset
133+
# )
134+
print("\n=== 训练 ===")
135+
trainer.train()
136+
test_results = trainer.evaluate()
137+
print(f"Test Accuracy: {test_results['eval_accuracy']:.4f}")
138+
139+
print("\n=== 训练后测试 ===")
140+
pred_after = test_single_image(model, processor, test_image)
141+
print(f"真实标签: {test_label}")
142+
print(f"预测标签: {pred_after}")

‎llm/finetune/bit/mindnlplog.txt

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
(MindSpore) [ma-user work]$python mindNLP_Bit_flowers.py
2+
Building prefix dict from the default dictionary ...
3+
Loading model from cache /tmp/jieba.cache
4+
Loading model cost 1.241 seconds.
5+
Prefix dict has been built successfully.
6+
Some weights of BitForImageClassification were not initialized from the model checkpoint at HorcruxNo13/bit-50 and are newly initialized because the shapes did not match:
7+
- classifier.1.weight: found shape (1000, 2048) in the checkpoint and (102, 2048) in the model instantiated
8+
- classifier.1.bias: found shape (1000,) in the checkpoint and (102,) in the model instantiated
9+
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
10+
test-00000-of-00006.parquet: 100%|█████████████████████████████████████████████████████████████████| 420M/420M [02:14<00:00, 3.12MB/s]
11+
test-00001-of-00006.parquet: 100%|█████████████████████████████████████████████████████████████████| 416M/416M [02:11<00:00, 3.17MB/s]
12+
test-00002-of-00006.parquet: 0%| | 0.00/429M [00:00<?, ?B/s]Error while downloading from https://cdn-lfs-us-1.hf-mirror.com/repos/b0/d3/b0d3d68b388c3ee41777a414af8253d880c3bb39c69b5fb303194abceea8e81f/68bf7479a332fe74bd6cf9066509d536768603058b539d03ce49aa6b22902b83?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27test-00002-of-00006.parquet%3B+filename%3D%22test-00002-of-00006.parquet%22%3B&Expires=1742347838&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc0MjM0NzgzOH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmhmLmNvL3JlcG9zL2IwL2QzL2IwZDNkNjhiMzg4YzNlZTQxNzc3YTQxNGFmODI1M2Q4ODBjM2JiMzljNjliNWZiMzAzMTk0YWJjZWVhOGU4MWYvNjhiZjc0NzlhMzMyZmU3NGJkNmNmOTA2NjUwOWQ1MzY3Njg2MDMwNThiNTM5ZDAzY2U0OWFhNmIyMjkwMmI4Mz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=IfSMlxvtnSQpPGjDDIPtQDTpfZK8BcQ2k%7EkTBFk%7EPNYoXH6fJzHZ0VjINpY1zlhPxhp8G2SsD8oxcrAj-QHLA5ysWEp7OKCcyt4Fp5vGP8fpArEw2zd-oDCblRE3WVNpdM1d65-1TkmIxPRoWk%7EiVBpMjAlTagGJguLBiimPMiFHHJYH1Dohvp2D6AYv8cOiyx48hvW58xtFlMZOG0qd-ibzXK9aHoIDRs7FTixLHXDcR2W41MRiJULl18Q4bxnr3%7EpsftaA6xpJje3gS1Q8WXoxtZ5i%7EzoJNchBgvXrs2YAQ83IelrCGMl%7EsFWqPlNkBBnneNr4UBwMDIJB90H5NQ__&Key-Pair-Id=K24J24Z295AEI9: HTTPSConnectionPool(host='cdn-lfs-us-1.hf-mirror.com', port=443): Read timed out.
13+
Trying to resume download...
14+
test-00002-of-00006.parquet: 100%|█████████████████████████████████████████████████████████████████| 429M/429M [02:38<00:00, 2.71MB/s]
15+
test-00002-of-00006.parquet: 0%| | 0.00/429M [02:50<?, ?B/s]
16+
test-00003-of-00006.parquet: 100%|█████████████████████████████████████████████████████████████████| 412M/412M [02:25<00:00, 2.83MB/s]
17+
test-00004-of-00006.parquet: 100%|█████████████████████████████████████████████████████████████████| 426M/426M [02:19<00:00, 3.05MB/s]
18+
test-00005-of-00006.parquet: 100%|█████████████████████████████████████████████████████████████████| 418M/418M [02:27<00:00, 2.83MB/s]
19+
validation-00000-of-00001.parquet: 100%|███████████████████████████████████████████████████████████| 416M/416M [02:26<00:00, 2.84MB/s]
20+
Generating train split: 100%|█████████████████████████████████████████████████████████████| 1020/1020 [00:01<00:00, 717.56 examples/s]
21+
Generating test split: 100%|██████████████████████████████████████████████████████████████| 6149/6149 [00:08<00:00, 710.30 examples/s]
22+
Generating validation split: 100%|████████████████████████████████████████████████████████| 1020/1020 [00:01<00:00, 819.73 examples/s]
23+
Saving the dataset (1/1 shards): 100%|██████████████████████████████████████████████████████| 816/816 [00:02<00:00, 338.37 examples/s]
24+
Saving the dataset (1/1 shards): 100%|██████████████████████████████████████████████████████| 204/204 [00:00<00:00, 221.10 examples/s]
25+
DatasetDict({
26+
train: Dataset({
27+
features: ['image', 'label'],
28+
num_rows: 816
29+
})
30+
test: Dataset({
31+
features: ['image', 'label'],
32+
num_rows: 204
33+
})
34+
})
35+
36+
=== 训练参数 ===
37+
38+
=== 先生成np数据 ===
39+
40+
=== 将预处理后的数据集转换为MindSpore格式 ===
41+
42+
=== 训练前测试 ===
43+
.真实标签: 26
44+
预测标签: 25
45+
Downloading builder script: 4.20kB [00:00, 11.8MB/s]
46+
47+
=== 创建Trainer实例 ===
48+
49+
=== 训练 ===
50+
0%| | 0/120 [00:00<?, ?it/s] 1%|▊ | 1/120 [00:09<18:09, 9.15s/it]{'eval_loss': 3.5184175968170166, 'eval_accuracy': 0.2734375, 'eval_runtime': 1.0791, 'eval_samples_per_second': 0.927, 'eval_steps_per_second': 0.927, 'epoch': 1.0}
51+
{'eval_loss': 1.7758612632751465, 'eval_accuracy': 0.75, 'eval_runtime': 0.998, 'eval_samples_per_second': 1.002, 'eval_steps_per_second': 1.002, 'epoch': 2.0}
52+
{'eval_loss': 0.9314232468605042, 'eval_accuracy': 0.875, 'eval_runtime': 0.9619, 'eval_samples_per_second': 1.04, 'eval_steps_per_second': 1.04, 'epoch': 3.0}
53+
{'eval_loss': 0.6095938682556152, 'eval_accuracy': 0.8984375, 'eval_runtime': 0.9827, 'eval_samples_per_second': 1.018, 'eval_steps_per_second': 1.018, 'epoch': 4.0}
54+
{'loss': 1.7124, 'learning_rate': 2.916666666666667e-05, 'epoch': 4.17}
55+
42%|████████████████████████████████████████ | 50/120 [09:18<11:06, 9.52s/it]{'eval_loss': 0.4878421127796173, 'eval_accuracy': 0.90625, 'eval_runtime': 0.9954, 'eval_samples_per_second': 1.005, 'eval_steps_per_second': 1.005, 'epoch': 5.0}
56+
{'eval_loss': 0.4401741027832031, 'eval_accuracy': 0.90625, 'eval_runtime': 0.9954, 'eval_samples_per_second': 1.005, 'eval_steps_per_second': 1.005, 'epoch': 6.0}
57+
{'eval_loss': 0.42397767305374146, 'eval_accuracy': 0.921875, 'eval_runtime': 1.0152, 'eval_samples_per_second': 0.985, 'eval_steps_per_second': 0.985, 'epoch': 7.0}
58+
{'eval_loss': 0.4162144362926483, 'eval_accuracy': 0.921875, 'eval_runtime': 0.9384, 'eval_samples_per_second': 1.066, 'eval_steps_per_second': 1.066, 'epoch': 8.0}
59+
{'loss': 0.0363, 'learning_rate': 8.333333333333334e-06, 'epoch': 8.33}
60+
{'eval_loss': 0.4113974869251251, 'eval_accuracy': 0.921875, 'eval_runtime': 0.9942, 'eval_samples_per_second': 1.006, 'eval_steps_per_second': 1.006, 'epoch': 9.0}
61+
{'eval_loss': 0.40957605838775635, 'eval_accuracy': 0.921875, 'eval_runtime': 1.394, 'eval_samples_per_second': 0.717, 'eval_steps_per_second': 0.717, 'epoch': 10.0}
62+
{'train_runtime': 1194.294, 'train_samples_per_second': 6.431, 'train_steps_per_second': 0.1, 'train_loss': 0.7326235515375932, 'epoch': 10.0}
63+
100%|███████████████████████████████████████████████████████████████████████████████████████████████| 120/120 [19:54<00:00, 9.95s/it]
64+
100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 3.53it/s]
65+
Test Accuracy: 0.9219
66+
67+
=== 训练后测试 ===
68+
真实标签: 26
69+
预测标签: 26

0 commit comments

Comments
 (0)