vectorplasticity commited on
Commit
4327377
·
verified ·
1 Parent(s): 52aeb6a

Add utils package init

Browse files
Files changed (1) hide show
  1. app/utils/__init__.py +59 -0
app/utils/__init__.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Utilities Package
3
+
4
+ Helper functions for training, datasets, models, and PEFT/LoRA.
5
+ """
6
+
7
+ from app.utils.training_utils import (
8
+ get_model_class_for_task,
9
+ estimate_memory_requirements,
10
+ get_training_args,
11
+ get_peft_config,
12
+ get_data_collator,
13
+ compute_metrics_factory,
14
+ save_training_artifacts,
15
+ generate_job_id
16
+ )
17
+ from app.utils.dataset_utils import (
18
+ load_dataset,
19
+ validate_dataset,
20
+ tokenize_dataset,
21
+ prepare_dataset_for_task
22
+ )
23
+ from app.utils.model_utils import (
24
+ load_model,
25
+ load_tokenizer,
26
+ apply_peft,
27
+ estimate_parameters
28
+ )
29
+ from app.utils.peft_utils import (
30
+ configure_lora,
31
+ configure_adalora,
32
+ apply_peft_config
33
+ )
34
+
35
+ __all__ = [
36
+ # Training utils
37
+ "get_model_class_for_task",
38
+ "estimate_memory_requirements",
39
+ "get_training_args",
40
+ "get_peft_config",
41
+ "get_data_collator",
42
+ "compute_metrics_factory",
43
+ "save_training_artifacts",
44
+ "generate_job_id",
45
+ # Dataset utils
46
+ "load_dataset",
47
+ "validate_dataset",
48
+ "tokenize_dataset",
49
+ "prepare_dataset_for_task",
50
+ # Model utils
51
+ "load_model",
52
+ "load_tokenizer",
53
+ "apply_peft",
54
+ "estimate_parameters",
55
+ # PEFT utils
56
+ "configure_lora",
57
+ "configure_adalora",
58
+ "apply_peft_config"
59
+ ]