Fix bug with safe suffix removal (#34)
Browse files- Safe suffix removal (507cf490804a20826b1defcf7f4dbf9af57a0f78)
Co-authored-by: Junkun Chen <shtpgshus@users.noreply.huggingface.co>
sample_finetune_speech.py
CHANGED
|
@@ -284,7 +284,7 @@ def evaluate(
|
|
| 284 |
for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
|
| 285 |
]
|
| 286 |
all_generated_texts.extend(generated_text)
|
| 287 |
-
labels = [processor.decode(_label_ids[_label_ids != 0]).
|
| 288 |
all_labels.extend(labels)
|
| 289 |
|
| 290 |
all_generated_texts = gather_object(all_generated_texts)
|
|
|
|
| 284 |
for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
|
| 285 |
]
|
| 286 |
all_generated_texts.extend(generated_text)
|
| 287 |
+
labels = [processor.decode(_label_ids[_label_ids != 0]).removesuffix(ANSWER_SUFFIX) for _label_ids in inputs["labels"]]
|
| 288 |
all_labels.extend(labels)
|
| 289 |
|
| 290 |
all_generated_texts = gather_object(all_generated_texts)
|