Update training init with finetune exports
Browse files
src/domain_tokenizer/training/__init__.py
CHANGED
|
@@ -3,11 +3,11 @@ Training utilities for domainTokenizer.
|
|
| 3 |
|
| 4 |
- data_pipeline: tokenize_user_sequences, pack_sequences, prepare_clm_dataset
|
| 5 |
- pretrain: pretrain_domain_model
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
|
| 8 |
-
from .data_pipeline import
|
| 9 |
-
tokenize_user_sequences,
|
| 10 |
-
pack_sequences,
|
| 11 |
-
prepare_clm_dataset,
|
| 12 |
-
)
|
| 13 |
from .pretrain import pretrain_domain_model
|
|
|
|
|
|
|
|
|
| 3 |
|
| 4 |
- data_pipeline: tokenize_user_sequences, pack_sequences, prepare_clm_dataset
|
| 5 |
- pretrain: pretrain_domain_model
|
| 6 |
+
- finetune_data: DomainFinetuneDataset, prepare_finetune_dataset
|
| 7 |
+
- finetune: finetune_domain_model
|
| 8 |
"""
|
| 9 |
|
| 10 |
+
from .data_pipeline import tokenize_user_sequences, pack_sequences, prepare_clm_dataset
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
from .pretrain import pretrain_domain_model
|
| 12 |
+
from .finetune_data import DomainFinetuneDataset, prepare_finetune_dataset
|
| 13 |
+
from .finetune import finetune_domain_model
|